| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from tests.utils import single_gpu_only |
| 7 | from vllm import SamplingParams |
| 8 | from vllm.config import CompilationConfig |
| 9 | |
| 10 | from ..utils import ( |
| 11 | assert_request_outputs_match, |
| 12 | evaluate_llm_for_gsm8k, |
| 13 | get_test_prompts, |
| 14 | ) |
| 15 | |
| 16 | |
| 17 | @pytest.mark.parametrize( |
| 18 | ["model_path", "expected_accuracy_threshold"], |
| 19 | [ |
| 20 | # Measured reference: 75%-80%. |
| 21 | ("RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3", 0.72), |
| 22 | # Measured reference: 87%-92%. |
| 23 | ("RedHatAI/Qwen3-8B-speculator.eagle3", 0.84), |
| 24 | ], |
| 25 | ids=["llama3_eagle3_speculator", "qwen3_eagle3_speculator"], |
| 26 | ) |
| 27 | @single_gpu_only |
| 28 | def test_speculators_model_integration( |
| 29 | monkeypatch: pytest.MonkeyPatch, |
| 30 | sampling_config: SamplingParams, |
| 31 | model_path: str, |
| 32 | expected_accuracy_threshold: float, |
| 33 | vllm_runner, |
| 34 | ): |
| 35 | """Test that speculators models work with the simplified integration. |
| 36 | |
| 37 | This verifies the `vllm serve <speculator-model>` use case where |
| 38 | speculative config is automatically detected from the model config |
| 39 | without requiring explicit --speculative-config argument. |
| 40 | |
| 41 | Tests: |
| 42 | 1. Speculator model is correctly detected |
| 43 | 2. Verifier model is extracted from speculator config |
| 44 | 3. Speculative decoding is automatically enabled |
| 45 | 4. Text generation works correctly |
| 46 | 5. GSM8k accuracy of the model passes a sanity check when speculative decoding on |
| 47 | 6. Output matches reference (non-speculative) generation |
| 48 | """ |
| 49 | monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") |
| 50 | |
| 51 | # Generate test prompts |
| 52 | test_prompts = get_test_prompts(mm_enabled=False) |
| 53 | |
| 54 | # First run: Direct speculator model (simplified integration) |
| 55 | with vllm_runner( |
| 56 | model_path, |
| 57 | block_size=None, |
| 58 | trust_remote_code=False, |
| 59 | enable_chunked_prefill=None, |
| 60 | compilation_config=CompilationConfig(), |
| 61 | max_model_len=4096, |
| 62 | gpu_memory_utilization=0.92, |
| 63 | ) as spec_runner: |
| 64 | evaluate_llm_for_gsm8k( |
| 65 | spec_runner.llm, expected_accuracy_threshold=expected_accuracy_threshold |
| 66 | ) |
| 67 | spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config) |
| 68 | |
| 69 | # Verify speculative config was auto-detected |
| 70 | assert spec_runner.llm.llm_engine.vllm_config.speculative_config is not None, ( |
| 71 | f"Speculative config should be auto-detected for {model_path}" |
| 72 | ) |
| 73 | |
| 74 | spec_config = spec_runner.llm.llm_engine.vllm_config.speculative_config |
| 75 | assert spec_config.num_speculative_tokens > 0, ( |
| 76 | f"Expected positive speculative tokens, " |
| 77 | f"got {spec_config.num_speculative_tokens}" |
| 78 | ) |
| 79 | |
| 80 | # Verify draft model is set to the speculator model |
| 81 | assert spec_config.model == model_path, ( |
| 82 | f"Draft model should be {model_path}, got {spec_config.model}" |
| 83 | ) |
| 84 | |
| 85 | # Extract verifier model for reference run |
| 86 | verifier_model = spec_runner.llm.llm_engine.vllm_config.model_config.model |
| 87 | |
| 88 | # Second run: Reference without speculative decoding |
| 89 | with vllm_runner( |
| 90 | verifier_model, |
| 91 | block_size=None, |
| 92 | trust_remote_code=False, |
| 93 | enable_chunked_prefill=None, |
| 94 | compilation_config=CompilationConfig(), |
| 95 | max_model_len=4096, |
| 96 | gpu_memory_utilization=0.92, |
| 97 | ) as ref_runner: |
| 98 | ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config) |
| 99 | |
| 100 | # Heuristic: expect at least 66% of prompts to match exactly |
| 101 | assert_request_outputs_match( |
| 102 | ref_outputs, |
| 103 | spec_outputs, |
| 104 | required_matches=int(0.66 * len(ref_outputs)), |
| 105 | context=f"speculator={model_path}, verifier={verifier_model}", |
| 106 | ) |