| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from unittest import mock |
| 5 | |
| 6 | import pytest |
| 7 | import torch |
| 8 | import torch.nn as nn |
| 9 | |
| 10 | from tests.v1.attention.utils import ( |
| 11 | BatchSpec, |
| 12 | create_common_attn_metadata, |
| 13 | create_standard_kv_cache_spec, |
| 14 | try_get_attention_backend, |
| 15 | ) |
| 16 | from vllm.config import ( |
| 17 | CacheConfig, |
| 18 | DeviceConfig, |
| 19 | ModelConfig, |
| 20 | ParallelConfig, |
| 21 | SchedulerConfig, |
| 22 | SpeculativeConfig, |
| 23 | VllmConfig, |
| 24 | ) |
| 25 | from vllm.config.load import LoadConfig |
| 26 | from vllm.model_executor.models.llama import LlamaForCausalLM |
| 27 | from vllm.platforms import current_platform |
| 28 | from vllm.v1.attention.backends.registry import AttentionBackendEnum |
| 29 | from vllm.v1.spec_decode.eagle import EagleProposer |
| 30 | |
| 31 | mimo_7b_dir = "XiaomiMiMo/MiMo-7B-Base" |
| 32 | DEVICE_TYPE = current_platform.device_type |
| 33 | |
| 34 | |
| 35 | def test_glm_mtp_defers_lm_head(default_vllm_config): |
| 36 | from vllm.model_executor.models import deepseek_mtp |
| 37 | from vllm.models.glm5next.common import mtp |
| 38 | |
| 39 | config = mock.MagicMock( |
| 40 | hidden_size=16, |
| 41 | rms_norm_eps=1e-5, |
| 42 | index_topk=8, |
| 43 | index_kpool=4, |
| 44 | ) |
| 45 | vllm_config = mock.MagicMock() |
| 46 | vllm_config.speculative_config.draft_model_config.hf_text_config = config |
| 47 | vllm_config.scheduler_config.max_num_batched_tokens = 4 |
| 48 | |
| 49 | with ( |
| 50 | mock.patch.object(mtp, "Glm5NextDecoderLayer", return_value=nn.Identity()), |
| 51 | mock.patch.object(deepseek_mtp, "ParallelLMHead") as parallel_lm_head, |
| 52 | mock.patch.object(mtp.current_platform, "device_type", "cpu"), |
| 53 | ): |
| 54 | layer = mtp.Glm5NextMultiTokenPredictorLayer(vllm_config, "model.layers.1") |
| 55 | |
| 56 | parallel_lm_head.assert_not_called() |
| 57 | assert layer.shared_head.head is None |
| 58 | |
| 59 | |
| 60 | def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer: |
| 61 | """Create an MTP proposer with unified model configuration.""" |
| 62 | model_config = ModelConfig( |
| 63 | model=mimo_7b_dir, runner="generate", max_model_len=100, trust_remote_code=True |
| 64 | ) |
| 65 | |
| 66 | speculative_config = SpeculativeConfig( |
| 67 | target_model_config=model_config, |
| 68 | target_parallel_config=ParallelConfig(), |
| 69 | model=mimo_7b_dir, |
| 70 | method="mtp", |
| 71 | num_speculative_tokens=num_speculative_tokens, |
| 72 | ) |
| 73 | |
| 74 | vllm_config = VllmConfig( |
| 75 | model_config=model_config, |
| 76 | cache_config=CacheConfig(), |
| 77 | speculative_config=speculative_config, |
| 78 | device_config=DeviceConfig(device=DEVICE_TYPE), |
| 79 | parallel_config=ParallelConfig(), |
| 80 | load_config=LoadConfig(), |
| 81 | scheduler_config=SchedulerConfig( |
| 82 | max_model_len=model_config.max_model_len, |
| 83 | is_encoder_decoder=model_config.is_encoder_decoder, |
| 84 | ), |
| 85 | ) |
| 86 | |
| 87 | return EagleProposer(vllm_config=vllm_config, device=DEVICE_TYPE) |
| 88 | |
| 89 | |
| 90 | @mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_pp_group") |
| 91 | @mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_layers_from_vllm_config") |
| 92 | @mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_model") |
| 93 | def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_group): |
| 94 | """Test MTP-specific model loading with unified model approach.""" |
| 95 | # Setup mocks |
| 96 | mock_model = mock.MagicMock() |
| 97 | mock_model.model.embed_tokens.weight.shape = (131072, 4096) |
| 98 | mock_get_model.return_value = mock_model |
| 99 | # MTP does not have its own embed_tokens or lm_head |
| 100 | # so it should share them with the target model |
| 101 | mock_model.has_own_embed_tokens = False |
| 102 | mock_model.has_own_lm_head = False |
| 103 | |
| 104 | target_attn_layers = {"target_attn_1": mock.MagicMock()} |
| 105 | all_attn_layers = {**target_attn_layers, "draft_attn_1": mock.MagicMock()} |
| 106 | target_indexer_layers: dict = {} |
| 107 | all_indexer_layers: dict = {} |
| 108 | |
| 109 | mock_get_layers.side_effect = [ |
| 110 | target_attn_layers, |
| 111 | target_indexer_layers, |
| 112 | all_attn_layers, |
| 113 | all_indexer_layers, |
| 114 | ] |
| 115 | |
| 116 | mock_pp_group = mock.MagicMock() |
| 117 | mock_pp_group.world_size = 1 |
| 118 | mock_get_pp_group.return_value = mock_pp_group |
| 119 | |
| 120 | # Create target model |
| 121 | class _TargetModelStub(LlamaForCausalLM): |
| 122 | model: mock.MagicMock |
| 123 | lm_head: mock.MagicMock |
| 124 | |
| 125 | target_model = mock.create_autospec(_TargetModelStub, instance=True) |
| 126 | target_model.model = mock.MagicMock() |
| 127 | target_model.model.embed_tokens.weight.shape = (131072, 4096) |
| 128 | target_model.lm_head = mock.MagicMock() |
| 129 | |
| 130 | # Create MTP proposer |
| 131 | proposer = _create_mtp_proposer(num_speculative_tokens=4) |
| 132 | proposer.load_model(target_model) |
| 133 | |
| 134 | # Verify MTP-specific behavior: |
| 135 | # Model is loaded |
| 136 | mock_get_model.assert_called_once() |
| 137 | # MTP shares lm_head with target model |
| 138 | assert proposer.model.lm_head == target_model.lm_head |
| 139 | # MTP shares embed_tokens with target model |
| 140 | assert proposer.model.model.embed_tokens == target_model.model.embed_tokens |
| 141 | |
| 142 | |
| 143 | @pytest.mark.parametrize("num_speculative_tokens", [1]) |
| 144 | def test_mtp_propose(num_speculative_tokens, monkeypatch): |
| 145 | """Test that MTP's forward method returns hidden states directly.""" |
| 146 | device = torch.device(DEVICE_TYPE) |
| 147 | batch_size = 2 |
| 148 | seq_lens = [5, 3] |
| 149 | total_tokens = sum(seq_lens) |
| 150 | vocab_size = 100 |
| 151 | |
| 152 | proposer = _create_mtp_proposer(num_speculative_tokens) |
| 153 | hidden_size = proposer.hidden_size |
| 154 | |
| 155 | # Mock the MTP model to verify it returns hidden states directly |
| 156 | model_mock = mock.MagicMock() |
| 157 | |
| 158 | # MTP returns hidden states directly |
| 159 | if num_speculative_tokens == 1: |
| 160 | model_mock.return_value = torch.zeros(total_tokens, hidden_size, device=device) |
| 161 | else: |
| 162 | # Multiple forward passes for multi-token speculation |
| 163 | forward_returns = [] |
| 164 | for i in range(num_speculative_tokens): |
| 165 | if i == 0: |
| 166 | h_states = torch.zeros(total_tokens, hidden_size, device=device) |
| 167 | else: |
| 168 | h_states = torch.zeros(batch_size, hidden_size, device=device) |
| 169 | forward_returns.append(h_states) |
| 170 | model_mock.side_effect = forward_returns |
| 171 | |
| 172 | # Mock compute_logits |
| 173 | def create_deterministic_logits(batch_size, vocab_size, token_offset): |
| 174 | logits = torch.full((batch_size, vocab_size), -100.0, device=device) |
| 175 | logits[:, token_offset] = 100.0 |
| 176 | return logits |
| 177 | |
| 178 | if num_speculative_tokens == 1: |
| 179 | model_mock.compute_logits.return_value = create_deterministic_logits( |
| 180 | batch_size, vocab_size, 42 |
| 181 | ) |
| 182 | else: |
| 183 | logits_returns = [ |
| 184 | create_deterministic_logits(batch_size, vocab_size, 42 + i) |
| 185 | for i in range(num_speculative_tokens) |
| 186 | ] |
| 187 | model_mock.compute_logits.side_effect = logits_returns |
| 188 | |
| 189 | proposer.model = model_mock |
| 190 | proposer._draft_attn_layer_names = {"layer.0"} |
| 191 | |
| 192 | # Prepare inputs |
| 193 | batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=seq_lens) |
| 194 | common_attn_metadata = create_common_attn_metadata( |
| 195 | batch_spec, block_size=16, device=device |
| 196 | ) |
| 197 | |
| 198 | target_token_ids = torch.randint(0, vocab_size, (total_tokens,), device=device) |
| 199 | target_positions = torch.cat( |
| 200 | [ |
| 201 | torch.arange(seq_lens[0], device=device), |
| 202 | torch.arange(seq_lens[1], device=device), |
| 203 | ] |
| 204 | ) |
| 205 | target_hidden_states = torch.randn(total_tokens, hidden_size, device=device) |
| 206 | next_token_ids = torch.randint( |
| 207 | 0, vocab_size, (batch_size,), dtype=torch.int32, device=device |
| 208 | ) |
| 209 | sampling_metadata = mock.MagicMock() |
| 210 | |
| 211 | # Setup attention metadata |
| 212 | attn_metadata_builder_cls, _ = try_get_attention_backend( |
| 213 | AttentionBackendEnum.FLASH_ATTN |
| 214 | ) |
| 215 | |
| 216 | attn_metadata_builder = attn_metadata_builder_cls( |
| 217 | kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config), |
| 218 | layer_names=list(proposer._draft_attn_layer_names), |
| 219 | vllm_config=proposer.vllm_config, |
| 220 | device=device, |
| 221 | ) |
| 222 | |
| 223 | mock_attn_group = mock.MagicMock() |
| 224 | mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder |
| 225 | mock_attn_group.layer_names = list(proposer._draft_attn_layer_names) |
| 226 | mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec |
| 227 | proposer.draft_attn_groups = [mock_attn_group] |
| 228 | |
| 229 | # Run propose |
| 230 | result = proposer.propose( |
| 231 | num_speculative_tokens=num_speculative_tokens, |
| 232 | target_token_ids=target_token_ids, |
| 233 | target_positions=target_positions, |
| 234 | target_hidden_states=target_hidden_states, |
| 235 | next_token_ids=next_token_ids, |
| 236 | token_indices_to_sample=None, |
| 237 | common_attn_metadata=common_attn_metadata, |
| 238 | sampling_metadata=sampling_metadata, |
| 239 | ) |
| 240 | |
| 241 | # Verify the model was called correctly |
| 242 | assert model_mock.called |
| 243 | # Verify output shape |
| 244 | assert result.shape == (batch_size, num_speculative_tokens) |