| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """DeepSeek V4.1 indexer backend for the DeepGEMM sparse MQA-logits path. |
| 4 | |
| 5 | Candidate-consuming indexer layers score only the candidate blocks published |
| 6 | by the candidate-source layer (``fp8_fp4_(paged_)sparse_mqa_logits``) instead |
| 7 | of dense logits over the whole context. This backend extends the dense |
| 8 | indexer metadata with the per-row scratch that path needs, so the layer op |
| 9 | allocates nothing per step, and lets all consumer layers of a step share one |
| 10 | candidate expansion and DeepGEMM schedule (they all read the same candidate |
| 11 | blocks). Opt in with ``AttentionConfig.indexer_sparse_logits``. |
| 12 | """ |
| 13 | |
| 14 | from dataclasses import dataclass, fields |
| 15 | |
| 16 | import torch |
| 17 | |
| 18 | from vllm.config import VllmConfig |
| 19 | from vllm.logger import init_logger |
| 20 | from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import ( |
| 21 | check_deep_select_layout, |
| 22 | has_deep_select, |
| 23 | pick_sparse_block_kv, |
| 24 | ) |
| 25 | from vllm.platforms import current_platform |
| 26 | from vllm.utils.deep_gemm import has_deep_gemm_sparse_mqa |
| 27 | from vllm.v1.attention.backend import CommonAttentionMetadata |
| 28 | from vllm.v1.attention.backends.mla.indexer import ( |
| 29 | DeepseekV32IndexerMetadata, |
| 30 | DeepseekV32IndexerMetadataBuilder, |
| 31 | DeepseekV41IndexerBackend, |
| 32 | dsa_indexer_uses_fp4, |
| 33 | ) |
| 34 | from vllm.v1.kv_cache_interface import AttentionSpec |
| 35 | |
| 36 | logger = init_logger(__name__) |
| 37 | |
| 38 | |
| 39 | @dataclass |
| 40 | class SparseMQARowsMetadata: |
| 41 | """Per-row state for one sparse-logits call: the decode rows, or one |
| 42 | prefill chunk. Buffers are builder-owned views sized to the rows. |
| 43 | |
| 44 | ``kernel_metadata`` (the DeepGEMM schedule) is filled by the first |
| 45 | consumer indexer layer of the step together with ``sparse_indices`` / |
| 46 | ``end``; later consumer layers reuse all three. |
| 47 | """ |
| 48 | |
| 49 | row_ks: torch.Tensor |
| 50 | """[rows] int32 K-range start per row; zeros for paged decode.""" |
| 51 | row_ke: torch.Tensor |
| 52 | """[rows] int32 K-range end (prefill) or compressed context length.""" |
| 53 | sparse_indices: torch.Tensor |
| 54 | """[rows, num_sparse_blocks] int32 DeepGEMM sparse block ids.""" |
| 55 | end: torch.Tensor |
| 56 | """[rows] int32 valid sparse-column count per row.""" |
| 57 | col_indices: torch.Tensor |
| 58 | """[rows, topk] int32 scratch for the sparse-column top-k.""" |
| 59 | block_table: torch.Tensor | None = None |
| 60 | """Decode only: [rows, pages] per-row page table.""" |
| 61 | row_indices: torch.Tensor | None = None |
| 62 | """Decode only: [rows] row -> request index.""" |
| 63 | kernel_metadata: torch.Tensor | None = None |
| 64 | |
| 65 | |
| 66 | @dataclass |
| 67 | class DeepseekV41SparseIndexerMetadata(DeepseekV32IndexerMetadata): |
| 68 | sparse_block_kv: int = 0 |
| 69 | sparse_decode: SparseMQARowsMetadata | None = None |
| 70 | sparse_prefill: list[SparseMQARowsMetadata] | None = None |
| 71 | """Parallel to ``prefill.chunks``.""" |
| 72 | |
| 73 | |
| 74 | class DeepseekV41SparseIndexerBackend(DeepseekV41IndexerBackend): |
| 75 | @staticmethod |
| 76 | def get_name() -> str: |
| 77 | return "DEEPSEEK_V41_SPARSE_INDEXER" |
| 78 | |
| 79 | @staticmethod |
| 80 | def get_builder_cls() -> type["DeepseekV41SparseIndexerMetadataBuilder"]: |
| 81 | return DeepseekV41SparseIndexerMetadataBuilder |
| 82 | |
| 83 | |
| 84 | class DeepseekV41SparseIndexerMetadataBuilder(DeepseekV32IndexerMetadataBuilder): |
| 85 | """Dense indexer metadata plus the sparse-logits row scratch. |
| 86 | |
| 87 | Every requirement of the sparse kernels is checked here, at engine start, |
| 88 | rather than falling back to the dense path per step. |
| 89 | """ |
| 90 | |
| 91 | def __init__( |
| 92 | self, |
| 93 | kv_cache_spec: AttentionSpec, |
| 94 | layer_names: list[str], |
| 95 | vllm_config: VllmConfig, |
| 96 | device: torch.device, |
| 97 | **kwargs, |
| 98 | ) -> None: |
| 99 | super().__init__(kv_cache_spec, layer_names, vllm_config, device, **kwargs) |
| 100 | hf_config = vllm_config.model_config.hf_text_config |
| 101 | candidate_block_size = getattr(hf_config, "candidate_block_size", 0) |
| 102 | candidate_topk_blocks = getattr(hf_config, "candidate_topk_blocks", 0) |
| 103 | topk_tokens = getattr(hf_config, "index_topk", 0) |
| 104 | prefix = "attention_config.indexer_sparse_logits" |
| 105 | if candidate_block_size <= 0 or candidate_topk_blocks <= 0: |
| 106 | raise ValueError( |
| 107 | f"{prefix} requires a two-level (candidate block) indexer model " |
| 108 | "such as DeepSeek-V4.1." |
| 109 | ) |
| 110 | if not dsa_indexer_uses_fp4(vllm_config): |
| 111 | raise ValueError( |
| 112 | f"{prefix} requires the MXFP4 indexer cache " |
| 113 | "(attention_config.indexer_kv_dtype='mxfp4'): the sparse kernels " |
| 114 | "take UE8M0-packed granularity-32 scales." |
| 115 | ) |
| 116 | if not ( |
| 117 | current_platform.is_cuda() |
| 118 | and current_platform.is_device_capability_family(100) |
| 119 | ): |
| 120 | raise ValueError(f"{prefix} requires an SM100-class GPU.") |
| 121 | if not has_deep_gemm_sparse_mqa(): |
| 122 | raise ValueError( |
| 123 | f"{prefix} requires DeepGEMM >= 2.8 " |
| 124 | "(fp8_fp4_sparse_mqa_logits not found)." |
| 125 | ) |
| 126 | if self.dcp_world_size > 1 or self.use_pcp: |
| 127 | raise NotImplementedError( |
| 128 | f"{prefix} is not supported with decode/prefill context parallel." |
| 129 | ) |
| 130 | if not self.supports_varlen: |
| 131 | raise ValueError( |
| 132 | f"{prefix} requires the varlen paged MQA logits decode layout." |
| 133 | ) |
| 134 | # The top-k runs on the kernels' bf16 logits with DeepSelect; there is |
| 135 | # no fp32 fallback, so the other top-k backends cannot be honored. |
| 136 | topk_backend = vllm_config.kernel_config.sparse_indexer_topk_backend |
| 137 | if not has_deep_select(): |
| 138 | raise ValueError( |
| 139 | f"{prefix} requires the DeepSelect top-k extension " |
| 140 | "(vllm._deepselect_C) for this GPU." |
| 141 | ) |
| 142 | if topk_backend not in ("auto", "deep_select"): |
| 143 | raise ValueError( |
| 144 | f"{prefix} uses DeepSelect for the top-k; " |
| 145 | f"kernel_config.sparse_indexer_topk_backend='{topk_backend}' " |
| 146 | "is not supported with it." |
| 147 | ) |
| 148 | self.sparse_block_kv = pick_sparse_block_kv(candidate_block_size) |
| 149 | if kv_cache_spec.block_size % self.sparse_block_kv != 0: |
| 150 | raise ValueError( |
| 151 | f"{prefix}: indexer cache pages ({kv_cache_spec.block_size} " |
| 152 | f"tokens) must be a multiple of {self.sparse_block_kv} tokens." |
| 153 | ) |
| 154 | self.num_sparse_blocks = candidate_topk_blocks * ( |
| 155 | candidate_block_size // self.sparse_block_kv |
| 156 | ) |
| 157 | num_sparse_cols = self.num_sparse_blocks * self.sparse_block_kv |
| 158 | check_deep_select_layout(num_sparse_cols, topk_tokens) |
| 159 | # Sparse logits are bf16 [rows, num_sparse_cols]; make the prefill |
| 160 | # chunker (which budgets fp32 [rows, seq_len]) size chunks as if every |
| 161 | # row were this wide. |
| 162 | self.min_split_seq_len = (num_sparse_cols * 2 + 3) // 4 |
| 163 | |
| 164 | # Rows are indexed by batch token (decode rows first, then prefill |
| 165 | # chunks by token range), like ``arange_buffer``. |
| 166 | max_rows = self.arange_buffer.shape[0] |
| 167 | int32 = dict(dtype=torch.int32, device=device) |
| 168 | self.sparse_indices_buffer = torch.zeros( |
| 169 | (max_rows, self.num_sparse_blocks), **int32 |
| 170 | ) |
| 171 | self.end_buffer = torch.zeros(max_rows, **int32) |
| 172 | self.col_indices_buffer = torch.zeros((max_rows, topk_tokens), **int32) |
| 173 | self.zero_row_ks = torch.zeros(max_rows, **int32) |
| 174 | logger.info_once( |
| 175 | "DeepSeek V4.1 indexer: candidate consumers score only the candidate " |
| 176 | "blocks with DeepGEMM sparse MQA logits (%d sparse blocks of %d " |
| 177 | "tokens per row).", |
| 178 | self.num_sparse_blocks, |
| 179 | self.sparse_block_kv, |
| 180 | ) |
| 181 | |
| 182 | def _prefill_split_seq_lens(self, seq_lens_cpu: torch.Tensor) -> torch.Tensor: |
| 183 | return seq_lens_cpu.clamp(min=self.min_split_seq_len) |
| 184 | |
| 185 | def _rows(self, start: int, end: int, row_ks, row_ke, **kwargs): |
| 186 | return SparseMQARowsMetadata( |
| 187 | row_ks=row_ks, |
| 188 | row_ke=row_ke, |
| 189 | sparse_indices=self.sparse_indices_buffer[start:end], |
| 190 | end=self.end_buffer[start:end], |
| 191 | col_indices=self.col_indices_buffer[start:end], |
| 192 | **kwargs, |
| 193 | ) |
| 194 | |
| 195 | def build( |
| 196 | self, |
| 197 | common_prefix_len: int, |
| 198 | common_attn_metadata: CommonAttentionMetadata, |
| 199 | fast_build: bool = False, |
| 200 | ) -> DeepseekV41SparseIndexerMetadata: |
| 201 | base = super().build(common_prefix_len, common_attn_metadata, fast_build) |
| 202 | metadata = DeepseekV41SparseIndexerMetadata( |
| 203 | **{f.name: getattr(base, f.name) for f in fields(base)}, |
| 204 | sparse_block_kv=self.sparse_block_kv, |
| 205 | ) |
| 206 | if base.decode is not None: |
| 207 | decode = base.decode |
| 208 | rows = decode.seq_lens.shape[0] |
| 209 | # The varlen layout flattens every query to one row, so the base |
| 210 | # decode tensors are already per row. |
| 211 | assert decode.seq_lens.shape[1] == 1 and not decode.requires_padding |
| 212 | assert decode.block_table.shape[0] == rows |
| 213 | metadata.sparse_decode = self._rows( |
| 214 | 0, |
| 215 | rows, |
| 216 | row_ks=self.zero_row_ks[:rows], |
| 217 | row_ke=decode.seq_lens.view(-1), |
| 218 | block_table=decode.block_table, |
| 219 | row_indices=( |
| 220 | decode.indices |
| 221 | if decode.indices is not None |
| 222 | else self.arange_buffer[:rows] |
| 223 | ), |
| 224 | ) |
| 225 | if base.prefill is not None: |
| 226 | metadata.sparse_prefill = [ |
| 227 | self._rows( |
| 228 | chunk.token_start, |
| 229 | chunk.token_end, |
| 230 | row_ks=chunk.cu_seqlen_ks, |
| 231 | row_ke=chunk.cu_seqlen_ke, |
| 232 | ) |
| 233 | for chunk in base.prefill.chunks |
| 234 | ] |
| 235 | return metadata |