| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Correctness tests for DSpark non-causal sliding-window MLA via sparse indices. |
| 4 | |
| 5 | DSpark drafts a block of N tokens whose attention is NON-CAUSAL within the block: |
| 6 | every block token attends to the sliding window of context AND to all block |
| 7 | tokens (including ones at later positions than itself). |
| 8 | |
| 9 | We can implement this using the existing sparse-MLA pathway by expanding the window size |
| 10 | to include the rest of the block tokens: instead of setting topk indices to the 127 |
| 11 | previous tokens, we expand it to the next power of 2 (256) and include up to |
| 12 | swa_size + block_size - 1 topk indices, so that each query attends to the rest. The |
| 13 | remaining slots are filled with padding. |
| 14 | |
| 15 | The sparse-MLA decode kernels (FlashMLA on SM90/SM100, FlashInfer TRTLLM on |
| 16 | SM100/SM120) are index-driven: each query attends over exactly the slots in its |
| 17 | index list, with no causal mask (see ``flash_mla_with_kvcache(..., indices=...)`` |
| 18 | and ``_forward_decode``'s "attend only by generated indices"). The existing |
| 19 | ``test_sparse_mla_backends`` suite already validates arbitrary index lists, but |
| 20 | only ones whose entries are <= the query's own position. This test suite specifically |
| 21 | ensures correctness of the non-causal attention case. |
| 22 | |
| 23 | This reuses the harness/helpers of ``test_sparse_mla_backends.py`` (same model |
| 24 | shapes, fp8_ds_mla round-trip, mock indexer, MockSparseMLAAttentionLayer); only |
| 25 | the index construction differs. |
| 26 | """ |
| 27 | |
| 28 | import math |
| 29 | from types import MethodType, SimpleNamespace |
| 30 | |
| 31 | import pytest |
| 32 | import torch |
| 33 | |
| 34 | from tests.v1.attention.test_mla_backends import ( |
| 35 | BatchSpec, |
| 36 | MockSparseMLAAttentionLayer, |
| 37 | create_and_prepopulate_kv_cache, |
| 38 | ) |
| 39 | from tests.v1.attention.test_sparse_mla_backends import ( |
| 40 | _quantize_dequantize_fp8_ds_mla, |
| 41 | ) |
| 42 | from tests.v1.attention.utils import ( |
| 43 | create_common_attn_metadata, |
| 44 | create_standard_kv_cache_spec, |
| 45 | create_vllm_config, |
| 46 | ) |
| 47 | from vllm.config import set_current_vllm_config |
| 48 | from vllm.model_executor.layers.linear import ColumnParallelLinear |
| 49 | from vllm.platforms import current_platform |
| 50 | |
| 51 | if not current_platform.is_cuda(): |
| 52 | pytest.skip( |
| 53 | "DSpark non-causal sparse MLA tests currently only support CUDA.", |
| 54 | allow_module_level=True, |
| 55 | ) |
| 56 | |
| 57 | from vllm.utils.math_utils import cdiv |
| 58 | from vllm.v1.attention.backend import AttentionCGSupport |
| 59 | from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( |
| 60 | FlashInferMLASparseTRTLLMBackend, |
| 61 | ) |
| 62 | from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend |
| 63 | from vllm.v1.attention.ops import flashmla |
| 64 | |
| 65 | DEVICE_TYPE = current_platform.device_type |
| 66 | |
| 67 | # (window, block_size, topk_width). topk_width must be a multiple of the kernel's |
| 68 | # B_TOPK (= padded query-head count, 64 or 128); we use 128-multiples to cover |
| 69 | # both. The "wide" case needs window + block > 128 -> width must grow past 128. |
| 70 | _DSPARK_CONFIGS = { |
| 71 | "small_block": (8, 4, 128), |
| 72 | "full_window_block": (128, 5, 256), |
| 73 | } |
| 74 | |
| 75 | |
| 76 | def _build_dspark_noncausal_indices( |
| 77 | seq_lens: list[int], |
| 78 | query_lens: list[int], |
| 79 | window: int, |
| 80 | topk_width: int, |
| 81 | device: torch.device, |
| 82 | ) -> torch.Tensor: |
| 83 | """Per-token sparse indices for the DSpark non-causal block. |
| 84 | |
| 85 | For a request with context length ``ctx`` and a query block of ``q_len`` |
| 86 | tokens (block positions ``ctx .. ctx+q_len-1``), EVERY block query attends to |
| 87 | the same set: the trailing ``window`` context positions plus all block |
| 88 | positions, i.e. the contiguous range ``[max(ctx-window,0) .. ctx+q_len-1]``. |
| 89 | This is non-causal: an early block query's list contains later block tokens |
| 90 | (future-pointing). The list is padded to ``topk_width`` with ``-1``. |
| 91 | """ |
| 92 | total_query_tokens = sum(query_lens) |
| 93 | sparse_indices = torch.full( |
| 94 | (total_query_tokens, topk_width), -1, dtype=torch.int32, device=device |
| 95 | ) |
| 96 | gt = 0 |
| 97 | for s_len, q_len in zip(seq_lens, query_lens): |
| 98 | ctx_len = s_len - q_len |
| 99 | lo = max(ctx_len - window, 0) |
| 100 | hi = ctx_len + q_len # exclusive: window context + the full block |
| 101 | idx_list = torch.arange(lo, hi, dtype=torch.int32, device=device) |
| 102 | n = idx_list.numel() |
| 103 | assert n <= topk_width, ( |
| 104 | f"index list ({n}) exceeds aligned topk width ({topk_width})" |
| 105 | ) |
| 106 | for _ in range(q_len): |
| 107 | sparse_indices[gt, :n] = idx_list |
| 108 | gt += 1 |
| 109 | return sparse_indices |
| 110 | |
| 111 | |
| 112 | def _run_sparse_backend_vs_sdpa( |
| 113 | backend_cls, |
| 114 | seq_lens: list[int], |
| 115 | query_lens: list[int], |
| 116 | sparse_indices: torch.Tensor, |
| 117 | kv_cache_dtype: str, |
| 118 | block_size: int, |
| 119 | num_heads: int, |
| 120 | device: torch.device, |
| 121 | force_future_dominance: bool = False, |
| 122 | qk_nope_head_dim: int = 128, |
| 123 | v_head_dim: int = 128, |
| 124 | stale_cpu_query_lens: list[int] | None = None, |
| 125 | ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: |
| 126 | """Run a sparse-MLA backend with the given per-token indices and compute a |
| 127 | dense per-token SDPA reference over the SAME indices. |
| 128 | |
| 129 | Mirrors ``test_sparse_mla_backends.test_sparse_backend_decode_correctness`` |
| 130 | but with externally-supplied (non-causal) ``sparse_indices``. |
| 131 | |
| 132 | ``num_heads`` selects the kernel's B_TOPK (= padded q-head count): 128 -> 128, |
| 133 | 64 -> 64. The aligned widths (128/256) are multiples of both, so num_heads=64 |
| 134 | exercises the head64 decode path that the SM100 alignment assert guards. |
| 135 | |
| 136 | ``force_future_dominance`` scales the LAST block token's latent KV so it |
| 137 | dominates the softmax for every query that attends to it. With random data the |
| 138 | few future block tokens carry negligible attention mass (especially with a wide |
| 139 | window), so causal and non-causal outputs coincide; this knob makes the |
| 140 | future-token contribution provably large for the differentiation test. It is |
| 141 | OFF for the correctness test (which needs sensitivity to all tokens). |
| 142 | |
| 143 | Returns (backend_output, noncausal_reference, causal_reference). The causal |
| 144 | reference restricts each query to indices <= its own absolute position. |
| 145 | """ |
| 146 | batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=query_lens) |
| 147 | topk_tokens = sparse_indices.shape[1] |
| 148 | dtype = torch.bfloat16 |
| 149 | use_fp8_ds_mla_quantization = kv_cache_dtype == "fp8_ds_mla" |
| 150 | |
| 151 | kv_lora_rank = 512 |
| 152 | qk_rope_head_dim = 64 |
| 153 | head_size = kv_lora_rank + qk_rope_head_dim |
| 154 | |
| 155 | max_seqlen = max(seq_lens) |
| 156 | total_cache_tokens = sum(seq_lens) |
| 157 | |
| 158 | vllm_config = create_vllm_config( |
| 159 | model_name="deepseek-ai/DeepSeek-V2-Lite-Chat", |
| 160 | tensor_parallel_size=1, |
| 161 | max_model_len=max_seqlen, |
| 162 | num_gpu_blocks=max(2048, cdiv(total_cache_tokens, block_size) + 1), |
| 163 | block_size=block_size, |
| 164 | hf_config_override={ |
| 165 | "index_topk": topk_tokens, |
| 166 | "attn_module_list_cfg": [{"topk_tokens": topk_tokens}], |
| 167 | }, |
| 168 | ) |
| 169 | model_config = vllm_config.model_config |
| 170 | model_config.hf_text_config = SimpleNamespace( |
| 171 | index_topk=topk_tokens, |
| 172 | q_lora_rank=None, |
| 173 | kv_lora_rank=kv_lora_rank, |
| 174 | qk_nope_head_dim=qk_nope_head_dim, |
| 175 | qk_rope_head_dim=qk_rope_head_dim, |
| 176 | v_head_dim=v_head_dim, |
| 177 | model_type="deepseek_v2", |
| 178 | ) |
| 179 | del model_config.hf_config.index_topk # Composite configs only nest this field. |
| 180 | model_config.dtype = dtype |
| 181 | model_config.get_num_attention_heads = MethodType( |
| 182 | lambda self, parallel_config: num_heads, model_config |
| 183 | ) |
| 184 | model_config.get_num_kv_heads = MethodType( |
| 185 | lambda self, parallel_config: 1, model_config |
| 186 | ) |
| 187 | model_config.get_head_size = MethodType(lambda self: head_size, model_config) |
| 188 | model_config.get_sliding_window = MethodType(lambda self: None, model_config) |
| 189 | |
| 190 | kv_cache_spec = create_standard_kv_cache_spec(vllm_config) |
| 191 | |
| 192 | torch.manual_seed(0) |
| 193 | scale = 1.0 / math.sqrt(head_size) |
| 194 | |
| 195 | # Shared MLA projection weights, used by both reference and backend. |
| 196 | W_UK = torch.rand( |
| 197 | kv_lora_rank, num_heads, qk_nope_head_dim, dtype=dtype, device=device |
| 198 | ) |
| 199 | W_UV = torch.rand(kv_lora_rank, num_heads, v_head_dim, dtype=dtype, device=device) |
| 200 | |
| 201 | all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], [] |
| 202 | kv_c_contexts, k_pe_contexts = [], [] |
| 203 | reference_outputs = [] |
| 204 | # Causal counterpart of the reference: same index lists, but each query is |
| 205 | # restricted to indices <= its own absolute position (drops future-pointing |
| 206 | # block tokens). Used to prove the non-causal result is genuinely different. |
| 207 | causal_reference_outputs = [] |
| 208 | |
| 209 | kv_cache_scale = torch.tensor(1.0, dtype=torch.float32, device=device) |
| 210 | global_token_idx = 0 |
| 211 | |
| 212 | for s_len, q_len in zip(seq_lens, query_lens): |
| 213 | ctx_len = s_len - q_len |
| 214 | |
| 215 | q_c = torch.rand( |
| 216 | q_len, |
| 217 | num_heads, |
| 218 | qk_nope_head_dim + qk_rope_head_dim, |
| 219 | dtype=dtype, |
| 220 | device=device, |
| 221 | ) |
| 222 | kv_c_full = torch.rand(s_len, kv_lora_rank, dtype=dtype, device=device) |
| 223 | k_pe_full = torch.rand(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device) |
| 224 | |
| 225 | if force_future_dominance: |
| 226 | # Scale the last block token's latent KV so its key/value dominate the |
| 227 | # softmax for any query attending to it. 4x in the latent dot makes its |
| 228 | # pre-softmax score exceed the others by a wide margin, so non-causal |
| 229 | # queries (which include it) diverge sharply from causal ones (which, |
| 230 | # for all but the last query, exclude it). Applied before quantization |
| 231 | # so cache and reference stay consistent. |
| 232 | kv_c_full[s_len - 1] = kv_c_full[s_len - 1] * 4.0 + 2.0 |
| 233 | |
| 234 | if use_fp8_ds_mla_quantization: |
| 235 | is_sm100 = torch.cuda.get_device_capability()[0] >= 10 |
| 236 | kv_c_full, k_pe_squeezed = _quantize_dequantize_fp8_ds_mla( |
| 237 | kv_c_full, |
| 238 | k_pe_full.squeeze(1), |
| 239 | block_size=block_size, |
| 240 | scale=kv_cache_scale, |
| 241 | simulate_sm100_e8m0_scales=is_sm100, |
| 242 | ) |
| 243 | k_pe_full = k_pe_squeezed.unsqueeze(1) |
| 244 | |
| 245 | q_nope, q_pe = q_c.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) |
| 246 | ql_nope = torch.einsum("qnh,lnh->qnl", q_nope, W_UK) |
| 247 | q_mqa = torch.cat([ql_nope, q_pe], dim=-1) |
| 248 | |
| 249 | k_mqa = torch.cat([kv_c_full, k_pe_full.squeeze(1)], dim=-1) |
| 250 | v_mqa = kv_c_full |
| 251 | |
| 252 | # Per-token sparse SDPA reference over the supplied (non-causal) indices. |
| 253 | def _sparse_sdpa(idx_tensor, q_tok, k_mqa=k_mqa, v_mqa=v_mqa): |
| 254 | k_sparse = k_mqa[idx_tensor].unsqueeze(1).expand(-1, num_heads, -1) |
| 255 | v_sparse = v_mqa[idx_tensor].unsqueeze(1).expand(-1, num_heads, -1) |
| 256 | out = torch.nn.functional.scaled_dot_product_attention( |
| 257 | q_tok.unsqueeze(0).transpose(1, 2), |
| 258 | k_sparse.unsqueeze(0).transpose(1, 2), |
| 259 | v_sparse.unsqueeze(0).transpose(1, 2), |
| 260 | scale=scale, |
| 261 | ) |
| 262 | out = out.transpose(1, 2).squeeze(0) |
| 263 | out = torch.einsum("qnl,lnv->qnv", out, W_UV) |
| 264 | return out.flatten(start_dim=-2) |
| 265 | |
| 266 | for q_idx in range(q_len): |
| 267 | tok_sparse_idx = sparse_indices[global_token_idx] |
| 268 | valid_indices = tok_sparse_idx[tok_sparse_idx >= 0].long() |
| 269 | |
| 270 | q_tok = q_mqa[q_idx : q_idx + 1] |
| 271 | reference_outputs.append(_sparse_sdpa(valid_indices, q_tok)) |
| 272 | |
| 273 | # Causal: drop indices pointing past this query's own position. |
| 274 | abs_pos = ctx_len + q_idx |
| 275 | causal_indices = valid_indices[valid_indices <= abs_pos] |
| 276 | causal_reference_outputs.append(_sparse_sdpa(causal_indices, q_tok)) |
| 277 | global_token_idx += 1 |
| 278 | |
| 279 | all_q_vllm.append(q_c) |
| 280 | all_kv_c_vllm.append(kv_c_full[ctx_len:]) |
| 281 | all_k_pe_vllm.append(k_pe_full[ctx_len:]) |
| 282 | kv_c_contexts.append(kv_c_full[: ctx_len + 1]) |
| 283 | k_pe_contexts.append(k_pe_full[: ctx_len + 1]) |
| 284 | |
| 285 | query_vllm = torch.cat(all_q_vllm, dim=0) |
| 286 | kv_c_vllm = torch.cat(all_kv_c_vllm, dim=0) |
| 287 | k_pe_vllm = torch.cat(all_k_pe_vllm, dim=0) |
| 288 | sdpa_reference = torch.cat(reference_outputs, dim=0) |
| 289 | causal_reference = torch.cat(causal_reference_outputs, dim=0) |
| 290 | |
| 291 | vllm_config.cache_config.cache_dtype = kv_cache_dtype |
| 292 | vllm_config.model_config.hf_text_config.index_topk = topk_tokens |
| 293 | |
| 294 | common_attn_metadata = create_common_attn_metadata( |
| 295 | batch_spec, block_size, device, arange_block_indices=True |
| 296 | ) |
| 297 | if stale_cpu_query_lens is not None: |
| 298 | # Adaptive verification updates only device boundaries; the stale CPU |
| 299 | # copy proves token_to_req_indices consumes the device layout. |
| 300 | assert len(stale_cpu_query_lens) == len(query_lens) |
| 301 | cpu_query_start_loc = [0] |
| 302 | for query_len in stale_cpu_query_lens: |
| 303 | cpu_query_start_loc.append(cpu_query_start_loc[-1] + query_len) |
| 304 | common_attn_metadata.query_start_loc_cpu = torch.tensor( |
| 305 | cpu_query_start_loc, dtype=torch.int32 |
| 306 | ) |
| 307 | kv_cache = create_and_prepopulate_kv_cache( |
| 308 | kv_c_contexts=kv_c_contexts, |
| 309 | k_pe_contexts=k_pe_contexts, |
| 310 | block_size=block_size, |
| 311 | head_size=head_size, |
| 312 | dtype=dtype, |
| 313 | device=device, |
| 314 | num_blocks=vllm_config.cache_config.num_gpu_blocks, |
| 315 | common_attn_metadata=common_attn_metadata, |
| 316 | randomize_blocks=False, |
| 317 | kv_cache_dtype=kv_cache_dtype, |
| 318 | scale=kv_cache_scale, |
| 319 | ) |
| 320 | |
| 321 | # The sparse builder clones the layer's dense-MHA prefill backend from |
| 322 | # static_forward_context; register a mock layer carrying one. |
| 323 | from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend |
| 324 | |
| 325 | prefill_backend = get_mla_prefill_backend(vllm_config)( |
| 326 | num_heads=num_heads, |
| 327 | scale=scale, |
| 328 | kv_lora_rank=kv_lora_rank, |
| 329 | qk_nope_head_dim=qk_nope_head_dim, |
| 330 | qk_rope_head_dim=qk_rope_head_dim, |
| 331 | v_head_dim=v_head_dim, |
| 332 | vllm_config=vllm_config, |
| 333 | ) |
| 334 | vllm_config.compilation_config.static_forward_context["placeholder"] = ( |
| 335 | SimpleNamespace(prefill_backend=prefill_backend) |
| 336 | ) |
| 337 | |
| 338 | builder = backend_cls.get_builder_cls()( |
| 339 | kv_cache_spec, ["placeholder"], vllm_config, device |
| 340 | ) |
| 341 | metadata = builder.build( |
| 342 | common_prefix_len=0, common_attn_metadata=common_attn_metadata |
| 343 | ) |
| 344 | |
| 345 | mock_indexer = SimpleNamespace(topk_indices_buffer=sparse_indices) |
| 346 | |
| 347 | kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1).view( |
| 348 | kv_lora_rank, num_heads * (qk_nope_head_dim + v_head_dim) |
| 349 | ) |
| 350 | mock_kv_b_proj = ColumnParallelLinear( |
| 351 | input_size=kv_lora_rank, |
| 352 | output_size=num_heads * (qk_nope_head_dim + v_head_dim), |
| 353 | bias=False, |
| 354 | ).to(device=device, dtype=dtype) |
| 355 | mock_kv_b_proj.weight = torch.nn.Parameter(kv_b_proj_weight.T.contiguous()) |
| 356 | |
| 357 | with set_current_vllm_config(vllm_config): |
| 358 | impl = backend_cls.get_impl_cls()( |
| 359 | num_heads=num_heads, |
| 360 | head_size=head_size, |
| 361 | scale=scale, |
| 362 | num_kv_heads=1, |
| 363 | alibi_slopes=None, |
| 364 | sliding_window=None, |
| 365 | kv_cache_dtype=vllm_config.cache_config.cache_dtype, |
| 366 | logits_soft_cap=None, |
| 367 | attn_type="decoder", |
| 368 | kv_sharing_target_layer_name=None, |
| 369 | q_lora_rank=None, |
| 370 | kv_lora_rank=kv_lora_rank, |
| 371 | qk_nope_head_dim=qk_nope_head_dim, |
| 372 | qk_rope_head_dim=qk_rope_head_dim, |
| 373 | qk_head_dim=qk_nope_head_dim + qk_rope_head_dim, |
| 374 | v_head_dim=v_head_dim, |
| 375 | kv_b_proj=mock_kv_b_proj, |
| 376 | indexer=mock_indexer, |
| 377 | ) |
| 378 | impl.process_weights_after_loading(dtype) |
| 379 | mock_layer = MockSparseMLAAttentionLayer( |
| 380 | impl=impl, |
| 381 | num_heads=num_heads, |
| 382 | qk_nope_head_dim=qk_nope_head_dim, |
| 383 | qk_rope_head_dim=qk_rope_head_dim, |
| 384 | v_head_dim=v_head_dim, |
| 385 | kv_lora_rank=kv_lora_rank, |
| 386 | device=device, |
| 387 | W_UK=W_UK, |
| 388 | W_UV=W_UV, |
| 389 | q_scale=1.0, |
| 390 | k_scale=1.0, |
| 391 | ) |
| 392 | |
| 393 | out_buffer = torch.empty( |
| 394 | metadata.num_actual_tokens, num_heads * v_head_dim, dtype=dtype, device=device |
| 395 | ) |
| 396 | with torch.inference_mode(): |
| 397 | backend_output = mock_layer.forward_impl( |
| 398 | query_vllm, kv_c_vllm, k_pe_vllm, kv_cache, metadata, out_buffer |
| 399 | ) |
| 400 | return backend_output, sdpa_reference, causal_reference |
| 401 | |
| 402 | |
| 403 | def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: int): |
| 404 | if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes: |
| 405 | pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}") |
| 406 | if ( |
| 407 | backend_cls is FlashMLASparseBackend |
| 408 | and kv_cache_dtype.startswith("fp8") |
| 409 | and kv_cache_dtype != "fp8_ds_mla" |
| 410 | ): |
| 411 | pytest.skip("FlashMLA Sparse fp8 only supports fp8_ds_mla kv-cache dtype") |
| 412 | if block_size not in backend_cls.get_supported_kernel_block_sizes(): |
| 413 | pytest.skip( |
| 414 | f"{backend_cls.get_name()} does not support block_size={block_size}" |
| 415 | ) |
| 416 | if backend_cls is FlashMLASparseBackend: |
| 417 | ok, reason = flashmla.is_flashmla_sparse_supported() |
| 418 | if not ok: |
| 419 | pytest.skip(reason) |
| 420 | elif backend_cls is FlashInferMLASparseTRTLLMBackend: |
| 421 | cap = current_platform.get_device_capability() |
| 422 | if cap is None or not backend_cls.supports_compute_capability(cap): |
| 423 | pytest.skip("FlashInferMLASparseTRTLLMBackend requires SM 10.x capability") |
| 424 | |
| 425 | |
| 426 | def test_flashinfer_sparse_mla_adaptive_varlen_matches_sdpa( |
| 427 | default_vllm_config, |
| 428 | dist_init, |
| 429 | workspace_init, |
| 430 | ): |
| 431 | """Adaptive request boundaries must drive SM100 sparse index conversion.""" |
| 432 | backend_cls = FlashInferMLASparseTRTLLMBackend |
| 433 | _skip_if_backend_unavailable(backend_cls, "fp8", 64) |
| 434 | assert ( |
| 435 | backend_cls.get_builder_cls().get_cudagraph_support(None, None) |
| 436 | == AttentionCGSupport.ALWAYS |
| 437 | ) |
| 438 | |
| 439 | device = torch.device(DEVICE_TYPE) |
| 440 | seq_lens = [257, 270, 265, 276] |
| 441 | query_lens = [1, 7, 3, 5] |
| 442 | sparse_indices = _build_dspark_noncausal_indices( |
| 443 | seq_lens, |
| 444 | query_lens, |
| 445 | window=128, |
| 446 | topk_width=256, |
| 447 | device=device, |
| 448 | ) |
| 449 | |
| 450 | backend_output, sdpa_reference, _ = _run_sparse_backend_vs_sdpa( |
| 451 | backend_cls, |
| 452 | seq_lens, |
| 453 | query_lens, |
| 454 | sparse_indices, |
| 455 | "fp8", |
| 456 | 64, |
| 457 | 16, |
| 458 | device, |
| 459 | qk_nope_head_dim=192, |
| 460 | v_head_dim=256, |
| 461 | stale_cpu_query_lens=[4, 4, 4, 4], |
| 462 | ) |
| 463 | |
| 464 | torch.testing.assert_close( |
| 465 | backend_output, |
| 466 | sdpa_reference, |
| 467 | rtol=0.065, |
| 468 | atol=0.05, |
| 469 | ) |
| 470 | |
| 471 | |
| 472 | @pytest.mark.parametrize( |
| 473 | "backend_cls", |
| 474 | [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend], |
| 475 | ids=["FlashMLA", "FlashInferTRTLLM"], |
| 476 | ) |
| 477 | @pytest.mark.parametrize("config_name", list(_DSPARK_CONFIGS.keys())) |
| 478 | # Per backend, the skip logic routes fp8 to the supported flavor: FlashMLA tests |
| 479 | # auto + fp8_ds_mla (and skips per-tensor "fp8", which it aliases to ds_mla); |
| 480 | # FlashInfer TRTLLM tests auto + per-tensor "fp8" (and skips fp8_ds_mla, which it |
| 481 | # does not implement). So both backends get a bf16 case and an fp8 case. |
| 482 | @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8_ds_mla", "fp8"]) |
| 483 | @pytest.mark.parametrize("block_size", [64]) |
| 484 | # h_q=128 -> B_TOPK=128; h_q=64 -> B_TOPK=64 (covers the head64 decode path the |
| 485 | # SM100 alignment assert specifically guards). Aligned widths (128/256) satisfy both. |
| 486 | @pytest.mark.parametrize("num_heads", [128, 64], ids=["h128", "h64"]) |
| 487 | def test_dspark_noncausal_sparse_mla_matches_sdpa( |
| 488 | default_vllm_config, |
| 489 | dist_init, |
| 490 | workspace_init, |
| 491 | backend_cls, |
| 492 | config_name, |
| 493 | kv_cache_dtype, |
| 494 | block_size, |
| 495 | num_heads, |
| 496 | ): |
| 497 | """Non-causal (window ∪ block, future-pointing) per-token indices must match |
| 498 | a dense SDPA reference over the same indices, for both sparse-MLA backends.""" |
| 499 | _skip_if_backend_unavailable(backend_cls, kv_cache_dtype, block_size) |
| 500 | |
| 501 | window, block, topk_width = _DSPARK_CONFIGS[config_name] |
| 502 | device = torch.device(DEVICE_TYPE) |
| 503 | |
| 504 | # Decode-style batch: each request has `block` query tokens and enough |
| 505 | # context for a full sliding window. |
| 506 | seq_lens = [window + block + 123, window + block + 50] |
| 507 | query_lens = [block, block] |
| 508 | |
| 509 | sparse_indices = _build_dspark_noncausal_indices( |
| 510 | seq_lens, query_lens, window, topk_width, device |
| 511 | ) |
| 512 | |
| 513 | # Sanity: the construction must actually be non-causal (an early block query |
| 514 | # must reference a later block position than itself). |
| 515 | ctx0 = seq_lens[0] - query_lens[0] |
| 516 | first_query_valid = sparse_indices[0][sparse_indices[0] >= 0] |
| 517 | assert int(first_query_valid.max()) >= ctx0 + query_lens[0] - 1, ( |
| 518 | "expected the first block query to attend to a future block token" |
| 519 | ) |
| 520 | |
| 521 | backend_output, sdpa_reference, _ = _run_sparse_backend_vs_sdpa( |
| 522 | backend_cls, |
| 523 | seq_lens, |
| 524 | query_lens, |
| 525 | sparse_indices, |
| 526 | kv_cache_dtype, |
| 527 | block_size, |
| 528 | num_heads, |
| 529 | device, |
| 530 | ) |
| 531 | |
| 532 | assert backend_output.shape == sdpa_reference.shape |
| 533 | assert backend_output.dtype == sdpa_reference.dtype |
| 534 | assert torch.isfinite(backend_output).all() |
| 535 | if kv_cache_dtype.startswith("fp8"): |
| 536 | rtol, atol = 0.065, 0.05 |
| 537 | else: |
| 538 | rtol, atol = 0.01, 0.01 |
| 539 | torch.testing.assert_close(backend_output, sdpa_reference, rtol=rtol, atol=atol) |
| 540 | |
| 541 | |
| 542 | @pytest.mark.parametrize( |
| 543 | "backend_cls", |
| 544 | [FlashMLASparseBackend, FlashInferMLASparseTRTLLMBackend], |
| 545 | ids=["FlashMLA", "FlashInferTRTLLM"], |
| 546 | ) |
| 547 | @pytest.mark.parametrize("config_name", list(_DSPARK_CONFIGS.keys())) |
| 548 | @pytest.mark.parametrize("block_size", [64]) |
| 549 | def test_dspark_noncausal_differs_from_causal( |
| 550 | default_vllm_config, |
| 551 | dist_init, |
| 552 | workspace_init, |
| 553 | backend_cls, |
| 554 | config_name, |
| 555 | block_size, |
| 556 | ): |
| 557 | """Differentiation guard: prove the backend genuinely attends to the |
| 558 | future-pointing indices (not silently applying a causal mask, and not merely |
| 559 | coinciding with a causal result because future tokens carry little weight). |
| 560 | |
| 561 | With random data the few future block tokens are a negligible fraction of the |
| 562 | attended set (especially with a wide window), so causal and non-causal outputs |
| 563 | are numerically indistinguishable -- that is correct physics, not a backend |
| 564 | bug. To make the check meaningful we use ``force_future_dominance`` so the last |
| 565 | block token dominates the softmax: the backend must then match the non-causal |
| 566 | reference and diverge sharply from the causal one. bf16 (``auto``) suffices; |
| 567 | the property is dtype-independent and fp8 correctness is covered above. |
| 568 | """ |
| 569 | _skip_if_backend_unavailable(backend_cls, "auto", block_size) |
| 570 | |
| 571 | window, block, topk_width = _DSPARK_CONFIGS[config_name] |
| 572 | device = torch.device(DEVICE_TYPE) |
| 573 | seq_lens = [window + block + 123, window + block + 50] |
| 574 | query_lens = [block, block] |
| 575 | |
| 576 | sparse_indices = _build_dspark_noncausal_indices( |
| 577 | seq_lens, query_lens, window, topk_width, device |
| 578 | ) |
| 579 | |
| 580 | backend_output, sdpa_reference, causal_reference = _run_sparse_backend_vs_sdpa( |
| 581 | backend_cls, |
| 582 | seq_lens, |
| 583 | query_lens, |
| 584 | sparse_indices, |
| 585 | "auto", |
| 586 | block_size, |
| 587 | 128, |
| 588 | device, |
| 589 | force_future_dominance=True, |
| 590 | ) |
| 591 | |
| 592 | # The two references must be clearly distinguishable for the check to mean |
| 593 | # anything (dominance guarantees this). |
| 594 | ref_gap = (sdpa_reference - causal_reference).abs().max().item() |
| 595 | assert ref_gap > 0.1, ( |
| 596 | f"non-causal and causal references are too close (gap={ref_gap}); " |
| 597 | "force_future_dominance did not create a separable scenario" |
| 598 | ) |
| 599 | |
| 600 | # Backend must track the NON-causal reference, not the causal one. |
| 601 | torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01) |
| 602 | causal_err = (backend_output - causal_reference).abs().max().item() |
| 603 | assert causal_err > 0.1, ( |
| 604 | f"non-causal backend output matches the causal reference " |
| 605 | f"(max abs diff={causal_err}); future-pointing indices are not attended to" |
| 606 | ) |
| 607 | |
| 608 | |
| 609 | @pytest.mark.parametrize("context_len", [20, 128, 900]) |
| 610 | @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn]) |
| 611 | @pytest.mark.parametrize("num_heads", [16, 64]) |
| 612 | def test_dsv41_flashinfer_dspark_window_matches_reference( |
| 613 | context_len, dtype, num_heads, monkeypatch |
| 614 | ): |
| 615 | """Draft queries see the full block without treating padded slots as keys.""" |
| 616 | if not current_platform.is_device_capability_family(100): |
| 617 | pytest.skip("DSV4 TRTLLM sparse attention requires SM100") |
| 618 | from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import ( |
| 619 | DeepseekSparseSWAFlashInferMetadataBuilder, |
| 620 | DeepseekV4FlashInferMLAAttention, |
| 621 | ) |
| 622 | from vllm.models.deepseek_v41.sparse_mla import ( |
| 623 | DeepseekV41SparseSWAMetadataBuilder, |
| 624 | ) |
| 625 | |
| 626 | torch.manual_seed(123) |
| 627 | device = "cuda" |
| 628 | query_lens = [5, 3] |
| 629 | context_lens = [context_len, context_len + 137] |
| 630 | num_real_tokens, num_tokens, head_dim = 8, 9, 512 |
| 631 | cache = torch.randn(32, 128, head_dim, device=device, dtype=torch.bfloat16).to( |
| 632 | dtype |
| 633 | ) |
| 634 | query = torch.randn( |
| 635 | num_tokens, num_heads, head_dim, device=device, dtype=torch.bfloat16 |
| 636 | ).to(dtype) |
| 637 | indices = torch.full((num_tokens, 256), -1, device=device, dtype=torch.int32) |
| 638 | visible_indices = [] |
| 639 | visible_lens = [] |
| 640 | for req, (context, query_len) in enumerate(zip(context_lens, query_lens)): |
| 641 | visible = ( |
| 642 | torch.arange(max(context - 128, 0), context + query_len, device=device) |
| 643 | + req * 16 * 128 |
| 644 | ) |
| 645 | visible_indices.extend([visible] * query_len) |
| 646 | visible_lens.extend([visible.numel()] * query_len) |
| 647 | for token, visible in enumerate(visible_indices): |
| 648 | indices[token, : visible.numel()] = visible.to(torch.int32) |
| 649 | query_start_loc = torch.tensor([0, 5, 8, 9], dtype=torch.int32) |
| 650 | metadata = SimpleNamespace( |
| 651 | num_decodes=3, |
| 652 | num_prefills=0, |
| 653 | num_decode_tokens=num_tokens, |
| 654 | num_prefill_tokens=0, |
| 655 | seq_lens=torch.tensor( |
| 656 | [context_lens[0] + 5, context_lens[1] + 3, 1], |
| 657 | device=device, |
| 658 | dtype=torch.int32, |
| 659 | ), |
| 660 | query_start_loc=query_start_loc.to(device), |
| 661 | query_start_loc_cpu=query_start_loc, |
| 662 | token_to_req_indices=torch.tensor( |
| 663 | [0] * 5 + [1] * 3 + [2], device=device, dtype=torch.int32 |
| 664 | ), |
| 665 | decode_swa_indices=indices, |
| 666 | decode_swa_width=256, |
| 667 | decode_swa_lens=torch.tensor( |
| 668 | visible_lens + [0], device=device, dtype=torch.int32 |
| 669 | ), |
| 670 | block_table=torch.arange(48, device=device, dtype=torch.int32).view(3, -1), |
| 671 | block_size=128, |
| 672 | replay_start=torch.zeros(3, device=device, dtype=torch.int32), |
| 673 | flashinfer_sparse_index_cache={}, |
| 674 | max_decode_query_len=5, |
| 675 | ) |
| 676 | |
| 677 | # Exercise FlashInfer preparation without constructing a model/config. |
| 678 | def init_parent(builder): |
| 679 | builder._max_tokens = num_tokens |
| 680 | builder.device = device |
| 681 | builder.window_size = 128 |
| 682 | |
| 683 | monkeypatch.setattr(DeepseekV41SparseSWAMetadataBuilder, "__init__", init_parent) |
| 684 | monkeypatch.setattr( |
| 685 | DeepseekV41SparseSWAMetadataBuilder, |
| 686 | "build", |
| 687 | lambda *args: metadata, |
| 688 | ) |
| 689 | builder = DeepseekSparseSWAFlashInferMetadataBuilder() |
| 690 | common_metadata = SimpleNamespace(causal=False) |
| 691 | builder.build(0, common_metadata) |
| 692 | prepared = ( |
| 693 | metadata.flashinfer_decode_topk_lens, |
| 694 | metadata.flashinfer_decode_seq_lens, |
| 695 | ) |
| 696 | attention = SimpleNamespace( |
| 697 | kv_cache_torch_dtype=dtype, |
| 698 | window_size=128, |
| 699 | compress_ratio=0, |
| 700 | topk_indices_buffer=torch.empty( |
| 701 | num_tokens, 0, device=device, dtype=torch.int32 |
| 702 | ), |
| 703 | scale=1 / math.sqrt(head_dim), |
| 704 | _flashinfer_fp8_bmm1_scale=1 / math.sqrt(head_dim), |
| 705 | _flashinfer_fp8_bmm2_scale=1.0, |
| 706 | attn_sink=None, |
| 707 | ) |
| 708 | attention._build_sparse_index_metadata = MethodType( |
| 709 | DeepseekV4FlashInferMLAAttention._build_sparse_index_metadata, attention |
| 710 | ) |
| 711 | output = torch.empty_like(query, dtype=torch.bfloat16) |
| 712 | |
| 713 | def forward(): |
| 714 | metadata.flashinfer_sparse_index_cache.clear() |
| 715 | DeepseekV4FlashInferMLAAttention._forward( |
| 716 | attention, query, None, cache, metadata, None, True, output |
| 717 | ) |
| 718 | |
| 719 | def check_output(): |
| 720 | references = [] |
| 721 | for token, visible in enumerate(visible_indices): |
| 722 | keys = cache.flatten(0, 1)[visible].float() |
| 723 | weights = torch.softmax( |
| 724 | query[token].float() @ keys.T / math.sqrt(head_dim), -1 |
| 725 | ) |
| 726 | references.append(weights @ keys) |
| 727 | atol = 0.01 if dtype == torch.bfloat16 else 0.05 |
| 728 | torch.testing.assert_close( |
| 729 | output[:num_real_tokens].float(), |
| 730 | torch.stack(references), |
| 731 | atol=atol, |
| 732 | rtol=0.05, |
| 733 | ) |
| 734 | |
| 735 | forward() |
| 736 | check_output() |
| 737 | graph = torch.cuda.CUDAGraph() |
| 738 | with torch.cuda.graph(graph): |
| 739 | forward() |
| 740 | # Replay must consume the current inputs, including KV written since capture. |
| 741 | query.copy_(torch.randn_like(query, dtype=torch.bfloat16).to(dtype)) |
| 742 | cache.copy_(torch.randn_like(cache, dtype=torch.bfloat16).to(dtype)) |
| 743 | # A new step changes visibility as well as Q/KV, without recapturing. |
| 744 | metadata.seq_lens.sub_(1) |
| 745 | metadata.decode_swa_lens[:num_real_tokens].sub_(1) |
| 746 | for token, visible in enumerate(visible_indices): |
| 747 | indices[token, visible.numel() - 1] = -1 |
| 748 | visible_indices[token] = visible[:-1] |
| 749 | builder.build(0, common_metadata) |
| 750 | assert metadata.flashinfer_decode_topk_lens.data_ptr() == prepared[0].data_ptr() |
| 751 | assert metadata.flashinfer_decode_seq_lens.data_ptr() == prepared[1].data_ptr() |
| 752 | torch.testing.assert_close(prepared[0], metadata.decode_swa_lens.clamp_min(128)) |
| 753 | torch.testing.assert_close( |
| 754 | prepared[1], metadata.seq_lens[metadata.token_to_req_indices.long()] |
| 755 | ) |
| 756 | graph.replay() |
| 757 | check_output() |