| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from typing import Any |
| 5 | |
| 6 | import pytest |
| 7 | |
| 8 | from vllm import SamplingParams |
| 9 | from vllm.config import CompilationConfig |
| 10 | from vllm.platforms import current_platform |
| 11 | |
| 12 | from ..utils import ( |
| 13 | _skip_if_insufficient_gpus_for_tp, |
| 14 | assert_request_outputs_match, |
| 15 | evaluate_llm_for_gsm8k, |
| 16 | get_test_prompts, |
| 17 | ) |
| 18 | |
| 19 | |
| 20 | def _run_eagle_correctness( |
| 21 | monkeypatch: pytest.MonkeyPatch, |
| 22 | sampling_config: SamplingParams, |
| 23 | model_setup: tuple[str, str, str, int], |
| 24 | mm_enabled: bool, |
| 25 | expected_accuracy_threshold: float, |
| 26 | enable_chunked_prefill: bool, |
| 27 | model_impl: str, |
| 28 | attn_backend: str, |
| 29 | vllm_runner, |
| 30 | ): |
| 31 | """Compare the outputs of an original LLM and a speculative LLM |
| 32 | which should be the same when using eagle speculative decoding. |
| 33 | """ |
| 34 | method, model_name, spec_model_name, tp_size = model_setup |
| 35 | _skip_if_insufficient_gpus_for_tp(tp_size) |
| 36 | |
| 37 | test_prompts = get_test_prompts(mm_enabled) |
| 38 | |
| 39 | extra_kwargs: dict[str, Any] = {} |
| 40 | if not mm_enabled and "Qwen3-VL" in model_name: |
| 41 | # These cases only exercise text generation. Avoid profiling an unused |
| 42 | # vision tower, which adds substantial memory to both reference runs. |
| 43 | extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0} |
| 44 | |
| 45 | if "Llama-4-Scout" in model_name and attn_backend == "FLASH_ATTN": |
| 46 | if current_platform.is_rocm(): |
| 47 | print( |
| 48 | "FLASH_ATTN for spec_decode not supported on " |
| 49 | "ROCm currently. Changing to FLEX_ATTENTION backend." |
| 50 | ) |
| 51 | attention_config = {"backend": "FLEX_ATTENTION"} |
| 52 | else: |
| 53 | attention_config = None |
| 54 | else: |
| 55 | attention_config = {"backend": attn_backend} |
| 56 | |
| 57 | if attn_backend == "TRITON_ATTN" and not current_platform.is_rocm(): |
| 58 | pytest.skip( |
| 59 | "TRITON_ATTN does not support " |
| 60 | "multi-token eagle spec decode on current platform" |
| 61 | ) |
| 62 | |
| 63 | with monkeypatch.context() as m: |
| 64 | m.setenv("VLLM_MLA_DISABLE", "1") |
| 65 | |
| 66 | if attn_backend == "ROCM_AITER_FA" and current_platform.is_rocm(): |
| 67 | if "deepseek" in model_name.lower(): |
| 68 | m.setenv("VLLM_ROCM_USE_AITER", "1") |
| 69 | m.delenv("VLLM_MLA_DISABLE", raising=False) |
| 70 | attention_config = {"backend": "ROCM_AITER_MLA"} |
| 71 | else: |
| 72 | m.setenv("VLLM_ROCM_USE_AITER", "1") |
| 73 | |
| 74 | max_model_len = 2048 |
| 75 | max_num_batched_tokens = 128 if enable_chunked_prefill else max_model_len |
| 76 | |
| 77 | with vllm_runner( |
| 78 | model_name, |
| 79 | block_size=None, |
| 80 | trust_remote_code=False, |
| 81 | max_model_len=max_model_len, |
| 82 | tensor_parallel_size=tp_size, |
| 83 | attention_config=attention_config, |
| 84 | enable_chunked_prefill=None, |
| 85 | compilation_config=CompilationConfig(), |
| 86 | **extra_kwargs, |
| 87 | ) as ref_runner: |
| 88 | evaluate_llm_for_gsm8k( |
| 89 | ref_runner.llm, |
| 90 | expected_accuracy_threshold=expected_accuracy_threshold, |
| 91 | ) |
| 92 | ref_outputs = ref_runner.llm.chat(test_prompts, sampling_config) |
| 93 | |
| 94 | with vllm_runner( |
| 95 | model_name, |
| 96 | block_size=None, |
| 97 | trust_remote_code=True, |
| 98 | tensor_parallel_size=tp_size, |
| 99 | speculative_config={ |
| 100 | "method": method, |
| 101 | "model": spec_model_name, |
| 102 | "num_speculative_tokens": 3, |
| 103 | "max_model_len": max_model_len, |
| 104 | }, |
| 105 | max_model_len=max_model_len, |
| 106 | max_num_batched_tokens=max_num_batched_tokens, |
| 107 | enable_chunked_prefill=enable_chunked_prefill, |
| 108 | model_impl=model_impl, |
| 109 | attention_config=attention_config, |
| 110 | compilation_config=CompilationConfig(), |
| 111 | **extra_kwargs, |
| 112 | ) as spec_runner: |
| 113 | # EAGLE/EAGLE3 supports async scheduling by default. |
| 114 | has_async = ( |
| 115 | spec_runner.llm.llm_engine.vllm_config.scheduler_config.async_scheduling |
| 116 | ) |
| 117 | assert has_async, ( |
| 118 | f"Expected async scheduling for {method}: target={model_name}, " |
| 119 | f"draft={spec_model_name}, backend={attn_backend}; got {has_async}" |
| 120 | ) |
| 121 | evaluate_llm_for_gsm8k( |
| 122 | spec_runner.llm, |
| 123 | expected_accuracy_threshold=expected_accuracy_threshold, |
| 124 | ) |
| 125 | spec_outputs = spec_runner.llm.chat(test_prompts, sampling_config) |
| 126 | |
| 127 | assert_request_outputs_match( |
| 128 | ref_outputs, |
| 129 | spec_outputs, |
| 130 | required_matches=int(0.6 * len(ref_outputs)) + 1, |
| 131 | context=( |
| 132 | f"{method} target={model_name}, draft={spec_model_name}, " |
| 133 | f"backend={attn_backend}" |
| 134 | ), |
| 135 | ) |