| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for sync-free speculative decoding with async scheduling.""" |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | # Test configurations: (model, spec_model, method, num_spec_tokens, backend_env) |
| 8 | SPEC_DECODE_CONFIGS = [ |
| 9 | pytest.param( |
| 10 | "meta-llama/Llama-3.2-1B-Instruct", |
| 11 | "nm-testing/Llama3_2_1B_speculator.eagle3", |
| 12 | "eagle3", |
| 13 | 2, |
| 14 | id="eagle3-llama", |
| 15 | ), |
| 16 | pytest.param( |
| 17 | "eagle618/deepseek-v3-random", |
| 18 | "eagle618/eagle-deepseek-v3-random", |
| 19 | "eagle", |
| 20 | 2, |
| 21 | id="eagle-mla-deepseek", |
| 22 | ), |
| 23 | pytest.param( |
| 24 | "Qwen/Qwen3.5-0.8B-Base", |
| 25 | "Qwen/Qwen3.5-0.8B-Base", |
| 26 | "mtp", |
| 27 | 1, |
| 28 | id="mtp-qwen3_5-hybrid", |
| 29 | ), |
| 30 | ] |
| 31 | |
| 32 | |
| 33 | @pytest.mark.parametrize( |
| 34 | "model,spec_model,method,num_spec_tokens", |
| 35 | SPEC_DECODE_CONFIGS, |
| 36 | ) |
| 37 | def test_no_sync_with_spec_decode( |
| 38 | model: str, |
| 39 | spec_model: str, |
| 40 | method: str, |
| 41 | num_spec_tokens: int, |
| 42 | monkeypatch, |
| 43 | vllm_runner, |
| 44 | ): |
| 45 | """Test generation while the worker rejects unintended GPU-CPU syncs.""" |
| 46 | monkeypatch.setenv("VLLM_GPU_SYNC_CHECK", "error") |
| 47 | |
| 48 | from vllm import SamplingParams |
| 49 | from vllm.config import CompilationConfig |
| 50 | |
| 51 | # Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup |
| 52 | # and peaks well above the 18GB MIG slice used by one of the CI lanes. |
| 53 | # This test only exercises text generation, so the vision tower is |
| 54 | # never needed. |
| 55 | extra_kwargs: dict = {} |
| 56 | if "Qwen3.5" in model: |
| 57 | extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0} |
| 58 | |
| 59 | with vllm_runner( |
| 60 | model, |
| 61 | block_size=None, |
| 62 | trust_remote_code=False, |
| 63 | max_model_len=256, |
| 64 | speculative_config={ |
| 65 | "method": method, |
| 66 | "num_speculative_tokens": num_spec_tokens, |
| 67 | "model": spec_model, |
| 68 | }, |
| 69 | enforce_eager=True, |
| 70 | async_scheduling=True, |
| 71 | enable_chunked_prefill=None, |
| 72 | compilation_config=CompilationConfig(), |
| 73 | **extra_kwargs, |
| 74 | ) as runner: |
| 75 | llm = runner.llm |
| 76 | |
| 77 | # Assert async scheduling is actually active before running inference. |
| 78 | assert llm.llm_engine.vllm_config.scheduler_config.async_scheduling, ( |
| 79 | f"Expected async_scheduling=True for spec decode, got False. " |
| 80 | f"method={method}, target={model}, draft={spec_model}" |
| 81 | ) |
| 82 | |
| 83 | outputs = llm.generate( |
| 84 | ["Hello, my name is"], |
| 85 | SamplingParams(temperature=0, max_tokens=10), |
| 86 | ) |
| 87 | |
| 88 | assert len(outputs) == 1, ( |
| 89 | f"{method} target={model}: expected one request output, got {len(outputs)}" |
| 90 | ) |
| 91 | assert outputs[0].outputs, ( |
| 92 | f"{method} target={model}: request output has no completion candidates" |
| 93 | ) |
| 94 | assert outputs[0].outputs[0].text, ( |
| 95 | f"{method} target={model}: generated completion text is empty" |
| 96 | ) |