| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from types import SimpleNamespace |
| 5 | from unittest.mock import Mock |
| 6 | |
| 7 | import numpy as np |
| 8 | import pytest |
| 9 | import torch |
| 10 | |
| 11 | from tests.v1.attention.test_gdn_metadata_builder import ( |
| 12 | BLOCK_SIZE, |
| 13 | DEVICE, |
| 14 | _create_gdn_builder, |
| 15 | ) |
| 16 | from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata |
| 17 | from vllm.config.compilation import CUDAGraphMode |
| 18 | from vllm.platforms import current_platform |
| 19 | from vllm.v1.attention.backends.recoverssm_metadata import ( |
| 20 | RecoverSSMMetadata, |
| 21 | RecoverSSMPostprocessMetadata, |
| 22 | ) |
| 23 | from vllm.v1.worker.gpu.model_states import mamba_hybrid |
| 24 | from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState |
| 25 | from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState |
| 26 | |
| 27 | |
| 28 | def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None: |
| 29 | state = object.__new__(MambaHybridModelState) |
| 30 | state.vllm_config = SimpleNamespace(num_speculative_tokens=0) |
| 31 | state.model_config = SimpleNamespace(max_model_len=8192) |
| 32 | state._align_mode = False |
| 33 | state.recoverssm = None |
| 34 | |
| 35 | positions = torch.tensor([1536], dtype=torch.int64) |
| 36 | input_batch = SimpleNamespace( |
| 37 | num_reqs=1, |
| 38 | num_tokens=1, |
| 39 | num_reqs_after_padding=1, |
| 40 | num_tokens_after_padding=1, |
| 41 | query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(), |
| 42 | query_start_loc=torch.tensor([0, 1], dtype=torch.int32), |
| 43 | num_scheduled_tokens=torch.tensor([1], dtype=torch.int32), |
| 44 | max_query_len=None, |
| 45 | seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32), |
| 46 | seq_lens=torch.tensor([1537], dtype=torch.int32), |
| 47 | is_prefilling_np=torch.tensor([False]).numpy(), |
| 48 | prefill_runs_as_decode_np=None, |
| 49 | dcp_local_seq_lens=None, |
| 50 | positions=positions, |
| 51 | prompt_lens=torch.tensor([1024], dtype=torch.int32), |
| 52 | ) |
| 53 | expected_metadata = {"layer": object()} |
| 54 | build_attn_metadata = Mock(return_value=expected_metadata) |
| 55 | monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata) |
| 56 | |
| 57 | metadata = state.prepare_attn( |
| 58 | input_batch=input_batch, |
| 59 | cudagraph_mode=CUDAGraphMode.NONE, |
| 60 | block_tables=(), |
| 61 | slot_mappings=torch.empty(0, dtype=torch.int64), |
| 62 | attn_groups=[], |
| 63 | kv_cache_config=Mock(), |
| 64 | ) |
| 65 | |
| 66 | assert metadata is expected_metadata |
| 67 | assert build_attn_metadata.call_args.kwargs["positions"] is positions |
| 68 | |
| 69 | |
| 70 | def test_padded_prompt_tail_builds_as_spec_decode( |
| 71 | monkeypatch: pytest.MonkeyPatch, |
| 72 | ) -> None: |
| 73 | """A one-token prompt tail over prior state, padded with K placeholder |
| 74 | drafts (e.g. a P/D decode-node arrival), must reach the GDN builder as a |
| 75 | spec-decode row. Built as a prefill, the placeholder tokens are folded into |
| 76 | the recurrent state and can't be rolled back. |
| 77 | """ |
| 78 | k = 3 |
| 79 | state = object.__new__(MambaHybridModelState) |
| 80 | state.vllm_config = SimpleNamespace(num_speculative_tokens=k) |
| 81 | state.model_config = SimpleNamespace(max_model_len=8192) |
| 82 | state._align_mode = False |
| 83 | state.recoverssm = None |
| 84 | state.num_accepted_tokens_gpu = torch.ones(4, dtype=torch.int32) |
| 85 | |
| 86 | # A verify decode, a padded prompt tail (128 of 129 prompt tokens already |
| 87 | # computed), and a fresh prompt chunk of the same length. |
| 88 | query_lens = [k + 1] * 3 |
| 89 | seq_lens = [50, 128 + k + 1, k + 1] |
| 90 | is_prefilling = [False, True, True] |
| 91 | query_start_loc = np.array([0, 4, 8, 12], dtype=np.int32) |
| 92 | input_batch = SimpleNamespace( |
| 93 | num_reqs=3, |
| 94 | num_tokens=12, |
| 95 | num_reqs_after_padding=3, |
| 96 | num_tokens_after_padding=12, |
| 97 | idx_mapping=torch.arange(3), |
| 98 | query_start_loc_np=query_start_loc, |
| 99 | query_start_loc=torch.from_numpy(query_start_loc), |
| 100 | num_scheduled_tokens=np.array(query_lens, dtype=np.int32), |
| 101 | num_draft_tokens_per_req=np.array([k, k, 0], dtype=np.int32), |
| 102 | max_query_len=None, |
| 103 | seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), |
| 104 | seq_lens=torch.tensor(seq_lens, dtype=torch.int32), |
| 105 | is_prefilling_np=np.array(is_prefilling), |
| 106 | prefill_runs_as_decode_np=np.array([False, True, False]), |
| 107 | dcp_local_seq_lens=None, |
| 108 | positions=torch.zeros(12, dtype=torch.int64), |
| 109 | prompt_lens=None, |
| 110 | ) |
| 111 | build_attn_metadata = Mock(return_value={}) |
| 112 | monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata) |
| 113 | state.prepare_attn( |
| 114 | input_batch=input_batch, |
| 115 | cudagraph_mode=CUDAGraphMode.NONE, |
| 116 | block_tables=(), |
| 117 | slot_mappings=torch.empty(0, dtype=torch.int64), |
| 118 | attn_groups=[], |
| 119 | kv_cache_config=Mock(), |
| 120 | ) |
| 121 | mamba_metadata = build_attn_metadata.call_args.kwargs[ |
| 122 | "model_specific_attn_metadata" |
| 123 | ] |
| 124 | assert mamba_metadata.is_prefilling.tolist() == [False, False, True] |
| 125 | |
| 126 | builder = _create_gdn_builder(num_speculative_tokens=k) |
| 127 | common = create_common_attn_metadata( |
| 128 | BatchSpec(seq_lens=seq_lens, query_lens=query_lens), BLOCK_SIZE, DEVICE |
| 129 | ).replace(**mamba_metadata.get_extra_common_attn_kwargs(0, 3)) |
| 130 | meta = builder.build( |
| 131 | common_prefix_len=0, |
| 132 | common_attn_metadata=common, |
| 133 | num_accepted_tokens=mamba_metadata.num_accepted_tokens, |
| 134 | num_decode_draft_tokens_cpu=mamba_metadata.num_decode_draft_tokens_cpu, |
| 135 | ) |
| 136 | |
| 137 | # Only the fresh prompt chunk needs the prefill kernels. |
| 138 | assert meta.num_spec_decodes == 2 |
| 139 | assert meta.num_prefills == 1 |
| 140 | assert meta.num_prefill_tokens == k + 1 |
| 141 | |
| 142 | |
| 143 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 144 | @pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)]) |
| 145 | def test_postprocess_state_scalar_with_int32_mapping( |
| 146 | num_sampled: int, expected_value: int |
| 147 | ) -> None: |
| 148 | state = object.__new__(MambaHybridModelState) |
| 149 | state.num_accepted_tokens_gpu = torch.full( |
| 150 | (4,), 9, dtype=torch.int32, device="cuda" |
| 151 | ) |
| 152 | state._align_mode = False |
| 153 | state.recoverssm = None |
| 154 | state._mamba_ctx = None |
| 155 | idx_mapping = torch.tensor([2, -1, 0], dtype=torch.int32, device="cuda") |
| 156 | |
| 157 | state.postprocess_state(idx_mapping, num_sampled) |
| 158 | |
| 159 | expected = torch.tensor( |
| 160 | [expected_value, 9, expected_value, 9], dtype=torch.int32, device="cuda" |
| 161 | ) |
| 162 | torch.testing.assert_close(state.num_accepted_tokens_gpu, expected) |
| 163 | |
| 164 | |
| 165 | def test_recoverssm_commits_accepted_window_after_v2_sampling() -> None: |
| 166 | state = RecoverSSMState() |
| 167 | metadata = Mock(spec=RecoverSSMMetadata) |
| 168 | metadata.commit_recoverssm_state.return_value = None |
| 169 | num_sampled = torch.tensor([3, 1], dtype=torch.int32) |
| 170 | idx_mapping = torch.tensor([0, 1], dtype=torch.int32) |
| 171 | num_accepted_tokens = torch.ones(2, dtype=torch.int32) |
| 172 | group = SimpleNamespace(layer_names=["layer"]) |
| 173 | |
| 174 | state.record_step({"layer": metadata}, [[group]], for_capture=False) |
| 175 | state.commit_step( |
| 176 | num_sampled, |
| 177 | idx_mapping, |
| 178 | state_indices=None, |
| 179 | num_accepted_tokens=num_accepted_tokens, |
| 180 | ) |
| 181 | state.commit_step( |
| 182 | num_sampled, |
| 183 | idx_mapping, |
| 184 | state_indices=None, |
| 185 | num_accepted_tokens=num_accepted_tokens, |
| 186 | ) |
| 187 | |
| 188 | metadata.commit_recoverssm_state.assert_called_once_with(num_sampled) |
| 189 | |
| 190 | |
| 191 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 192 | def test_recoverssm_align_tracks_mixed_batch_state_and_neutralizes_copy_bias() -> None: |
| 193 | state = object.__new__(MambaHybridModelState) |
| 194 | state._align_mode = True |
| 195 | state._mamba_ctx = None |
| 196 | state._mamba_state_idx_gpu = torch.full((5,), -1, dtype=torch.int32, device="cuda") |
| 197 | state.recoverssm = RecoverSSMState() |
| 198 | state.num_accepted_tokens_gpu = torch.full( |
| 199 | (5,), 9, dtype=torch.int32, device="cuda" |
| 200 | ) |
| 201 | metadata = Mock(spec=RecoverSSMMetadata) |
| 202 | metadata.commit_recoverssm_state.return_value = RecoverSSMPostprocessMetadata( |
| 203 | num_spec_decodes=1, |
| 204 | request_indices=torch.tensor([1], dtype=torch.int32, device="cuda"), |
| 205 | num_computed_tokens=torch.tensor([6, 7], dtype=torch.int32, device="cuda"), |
| 206 | block_size=8, |
| 207 | block_table=torch.zeros((2, 4), dtype=torch.int32, device="cuda"), |
| 208 | ) |
| 209 | num_sampled = torch.tensor([2, 3], dtype=torch.int32, device="cuda") |
| 210 | idx_mapping = torch.tensor([3, 1], dtype=torch.int32, device="cuda") |
| 211 | group = SimpleNamespace(layer_names=["layer"]) |
| 212 | |
| 213 | state.recoverssm.record_step({"layer": metadata}, [[group]], for_capture=False) |
| 214 | |
| 215 | state.postprocess_state(idx_mapping, num_sampled) |
| 216 | |
| 217 | expected_state_indices = [-1, 1, -1, -1, -1] |
| 218 | assert state._mamba_state_idx_gpu.tolist() == expected_state_indices |
| 219 | expected_accepted = [9, 1, 9, 2, 9] |
| 220 | assert state.num_accepted_tokens_gpu.tolist() == expected_accepted |