A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 04:15 UTC
[Speculators] Move tests + fix integration (#27308)
Signed-off-by: Dipika Sikka <dipikasikka1@gmail.com> Signed-off-by: Rahul Tuli <rtuli@redhat.com> Signed-off-by: rahul-tuli <rtuli@redhat.com> Co-authored-by: Rahul Tuli <rtuli@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Dipika Sikka committed
Oct 29, 2025 at 03:54 UTC
413ef7a3b4d8722b8677f15e2320604a5a3d3b69
3 files changed
+97
-15
tests/v1/e2e/test_spec_decode.py
+80
@@ -121,6 +121,86 @@ def test_ngram_correctness(
121
cleanup_dist_env_and_memory()
122
123
124
+@pytest.mark.parametrize(
125
+ "model_path",
126
+ [
127
+ "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3",
128
+ "RedHatAI/Qwen3-8B-speculator.eagle3",
129
+ ],
130
+ ids=["llama3_eagle3_speculator", "qwen3_eagle3_speculator"],
131
+)
132
+def test_speculators_model_integration(
133
+ monkeypatch: pytest.MonkeyPatch,
134
+ sampling_config: SamplingParams,
135
+ model_path: str,
136
+):
137
+ """
138
+ Test that speculators models work with the simplified integration.
139
+
140
+ This verifies the `vllm serve <speculator-model>` use case where
141
+ speculative config is automatically detected from the model config
142
+ without requiring explicit --speculative-config argument.
143
+
144
+ Tests:
145
+ 1. Speculator model is correctly detected
146
+ 2. Verifier model is extracted from speculator config
147
+ 3. Speculative decoding is automatically enabled
148
+ 4. Text generation works correctly
149
+ 5. Output matches reference (non-speculative) generation
150
+ """
151
+ monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
152
+
153
+ # Generate test prompts
154
+ test_prompts = get_test_prompts(mm_enabled=False)
155
+
156
+ # First run: Direct speculator model (simplified integration)
157
+ spec_llm = LLM(model=model_path, max_model_len=1024)
158
+ spec_outputs = spec_llm.chat(test_prompts, sampling_config)
159
+
160
+ # Verify speculative config was auto-detected
161
+ assert spec_llm.llm_engine.vllm_config.speculative_config is not None, (
162
+ f"Speculative config should be auto-detected for {model_path}"
163
+ )
164
+
165
+ spec_config = spec_llm.llm_engine.vllm_config.speculative_config
166
+ assert spec_config.num_speculative_tokens > 0, (
167
+ f"Expected positive speculative tokens, "
168
+ f"got {spec_config.num_speculative_tokens}"
169
+ )
170
+
171
+ # Verify draft model is set to the speculator model
172
+ assert spec_config.model == model_path, (
173
+ f"Draft model should be {model_path}, got {spec_config.model}"
174
+ )
175
+
176
+ # Extract verifier model for reference run
177
+ verifier_model = spec_llm.llm_engine.vllm_config.model_config.model
178
+
179
+ del spec_llm
180
+ torch.cuda.empty_cache()
181
+ cleanup_dist_env_and_memory()
182
+
183
+ # Second run: Reference without speculative decoding
184
+ ref_llm = LLM(model=verifier_model, max_model_len=1024)
185
+ ref_outputs = ref_llm.chat(test_prompts, sampling_config)
186
+ del ref_llm
187
+ torch.cuda.empty_cache()
188
+ cleanup_dist_env_and_memory()
189
+
190
+ # Compare outputs
191
+ matches = sum(
192
+ 1
193
+ for ref, spec in zip(ref_outputs, spec_outputs)
194
+ if ref.outputs[0].text == spec.outputs[0].text
195
+ )
196
+
197
+ # Heuristic: expect at least 66% of prompts to match exactly
198
+ assert matches >= int(0.66 * len(ref_outputs)), (
199
+ f"Only {matches}/{len(ref_outputs)} outputs matched. "
200
+ f"Expected at least {int(0.66 * len(ref_outputs))} matches."
201
+ )
202
+
203
+
204
@pytest.mark.parametrize(
205
["model_setup", "mm_enabled"],
206
[
tests/v1/spec_decode/test_speculators_eagle3.py
renamed
-4
@@ -22,10 +22,6 @@ from vllm.model_executor.models.interfaces import supports_eagle3
22
"nm-testing/Speculator-Qwen3-8B-Eagle3-converted-071-quantized-w4a16",
23
id="qwen3-eagle3-speculator-w4a16-verifier",
24
),
25
- pytest.param(
26
- "nm-testing/random-weights-llama3.1.8b-2layer-eagle3",
27
- id="llama3-eagl3-multiple-layers",
28
- ),
25
],
26
)
27
def test_eagle3_speculators_model(
vllm/engine/arg_utils.py
+17
-11
@@ -81,7 +81,7 @@ from vllm.transformers_utils.config import (
81
is_interleaved,
82
maybe_override_with_speculators,
83
)
84
-from vllm.transformers_utils.utils import check_gguf_file
84
+from vllm.transformers_utils.utils import check_gguf_file, is_s3
85
from vllm.utils.argparse_utils import FlexibleArgumentParser
86
from vllm.utils.mem_constants import GiB_bytes
87
from vllm.utils.network_utils import get_ip
@@ -1305,20 +1305,26 @@ class EngineArgs:
1305
1306
device_config = DeviceConfig(device=cast(Device, current_platform.device_type))
1307
1308
+ # Check if the model is a speculator and override model/tokenizer/config
1309
+ # BEFORE creating ModelConfig, so the config is created with the target model
1310
+ # Skip speculator detection for S3 models since HuggingFace cannot load
1311
+ # configs directly from S3 URLs. S3 models can still use speculators with
1312
+ # explicit --speculative-config.
1313
+ if not is_s3(self.model):
1314
+ (self.model, self.tokenizer, self.speculative_config) = (
1315
+ maybe_override_with_speculators(
1316
+ model=self.model,
1317
+ tokenizer=self.tokenizer,
1318
+ revision=self.revision,
1319
+ trust_remote_code=self.trust_remote_code,
1320
+ vllm_speculative_config=self.speculative_config,
1321
+ )
1322
+ )
1323
+
1324
model_config = self.create_model_config()
1325
self.model = model_config.model
1326
self.tokenizer = model_config.tokenizer
1327
1312
- (self.model, self.tokenizer, self.speculative_config) = (
1313
- maybe_override_with_speculators(
1314
- model=self.model,
1315
- tokenizer=self.tokenizer,
1316
- revision=self.revision,
1317
- trust_remote_code=self.trust_remote_code,
1318
- vllm_speculative_config=self.speculative_config,
1319
- )
1320
- )
1321
-
1328
# * If VLLM_USE_V1 is unset, we enable V1 for "supported features"
1329
# and fall back to V0 for experimental or unsupported features.
1330
# * If VLLM_USE_V1=1, we enable V1 for supported + experimental