A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Config] add "qwen" as a native eagle3 target supported model (#22333)
Signed-off-by: lechen <lecself@163.com> Signed-off-by: LeChen <lecself@163.com>
Le Chen committed
Aug 10, 2025 at 11:21 UTC
3d7363e61c0a27bcba9e6694ae9771f9b780ce3d
5 files changed
+30
-27
tests/models/registry.py
+4
@@ -525,6 +525,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
525
trust_remote_code=True,
526
speculative_model="yuhuili/EAGLE3-LLaMA3.1-Instruct-8B",
527
tokenizer="meta-llama/Llama-3.1-8B-Instruct"),
528
+ "LlamaForCausalLMEagle3": _HfExamplesInfo("AngelSlim/Qwen3-8B_eagle3", # noqa: E501
529
+ trust_remote_code=True,
530
+ speculative_model="AngelSlim/Qwen3-8B_eagle3",
531
+ tokenizer="Qwen/Qwen3-8B"),
532
"EagleLlama4ForCausalLM": _HfExamplesInfo(
533
"morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct",
534
trust_remote_code=True,
tests/v1/e2e/test_spec_decode.py
+21
-18
@@ -125,24 +125,27 @@ def test_ngram_correctness(
125
cleanup_dist_env_and_memory()
126
127
128
-@pytest.mark.parametrize(
129
- ["model_setup", "mm_enabled"], [
130
- (("eagle", "meta-llama/Llama-3.1-8B-Instruct",
131
- "yuhuili/EAGLE-LLaMA3.1-Instruct-8B", 1), False),
132
- (("eagle3", "meta-llama/Llama-3.1-8B-Instruct",
133
- "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B", 1), False),
134
- pytest.param(
135
- ("eagle", "meta-llama/Llama-4-Scout-17B-16E-Instruct",
136
- "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct", 4),
137
- False,
138
- marks=pytest.mark.skip(reason="Skipping due to CI OOM issues")),
139
- pytest.param(
140
- ("eagle", "meta-llama/Llama-4-Scout-17B-16E-Instruct",
141
- "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct", 4),
142
- True,
143
- marks=pytest.mark.skip(reason="Skipping due to CI OOM issues")),
144
- ],
145
- ids=["llama3_eagle", "llama3_eagle3", "llama4_eagle", "llama4_eagle_mm"])
128
+@pytest.mark.parametrize(["model_setup", "mm_enabled"], [
129
+ (("eagle3", "Qwen/Qwen3-8B", "AngelSlim/Qwen3-8B_eagle3", 1), False),
130
+ (("eagle", "meta-llama/Llama-3.1-8B-Instruct",
131
+ "yuhuili/EAGLE-LLaMA3.1-Instruct-8B", 1), False),
132
+ (("eagle3", "meta-llama/Llama-3.1-8B-Instruct",
133
+ "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B", 1), False),
134
+ pytest.param(
135
+ ("eagle", "meta-llama/Llama-4-Scout-17B-16E-Instruct",
136
+ "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct", 4),
137
+ False,
138
+ marks=pytest.mark.skip(reason="Skipping due to CI OOM issues")),
139
+ pytest.param(
140
+ ("eagle", "meta-llama/Llama-4-Scout-17B-16E-Instruct",
141
+ "morgendave/EAGLE-Llama-4-Scout-17B-16E-Instruct", 4),
142
+ True,
143
+ marks=pytest.mark.skip(reason="Skipping due to CI OOM issues")),
144
+],
145
+ ids=[
146
+ "qwen3_eagle3", "llama3_eagle", "llama3_eagle3",
147
+ "llama4_eagle", "llama4_eagle_mm"
148
+ ])
149
@pytest.mark.parametrize("attn_backend",
150
get_attn_backend_list_based_on_platform())
151
def test_eagle_correctness(
vllm/config/__init__.py
+1
-7
@@ -2852,13 +2852,7 @@ class SpeculativeConfig:
2852
"speculative decoding is > 1, but got "
2853
f"{self.disable_by_batch_size=}")
2854
2855
- from vllm.transformers_utils.configs import SpeculatorsConfig
2856
-
2857
- eagle3_target_supported = ["llama"]
2858
- if self.draft_model_config and isinstance(
2859
- self.draft_model_config.hf_config, SpeculatorsConfig):
2860
- eagle3_target_supported.append("qwen")
2861
-
2855
+ eagle3_target_supported = ["llama", "qwen"]
2856
if self.method == "eagle3" and self.target_model_config and not any(
2857
supported_model in
2858
self.target_model_config.hf_text_config.model_type
vllm/model_executor/models/registry.py
+1
@@ -259,6 +259,7 @@ _SPECULATIVE_DECODING_MODELS = {
259
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
260
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
261
"Eagle3LlamaForCausalLM": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
262
+ "LlamaForCausalLMEagle3": ("llama_eagle3", "Eagle3LlamaForCausalLM"),
263
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
264
"Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
265
"MedusaModel": ("medusa", "Medusa"),
vllm/transformers_utils/configs/eagle.py
+3
-2
@@ -45,6 +45,7 @@ class EAGLEConfig(PretrainedConfig):
45
46
# Eagle model name should follow naming convention of
47
# LlamaForCausalLM -> EagleLlamaForCausalLM
48
+ # LlamaForCausalLM -> Eagle3LlamaForCausalLM / LlamaForCausalLMEagle3
49
if method == "eagle":
50
assert self.model is not None, \
51
"model should not be None when method is eagle"
@@ -56,8 +57,8 @@ class EAGLEConfig(PretrainedConfig):
57
assert self.model is not None, \
58
"model should not be None when method is eagle3"
59
kwargs["architectures"] = [
59
- f"Eagle3{arch}" if not arch.startswith("Eagle3") \
60
- else arch for arch in self.model.architectures
60
+ arch if arch.startswith("Eagle3") or arch.endswith("Eagle3")
61
+ else f"Eagle3{arch}" for arch in self.model.architectures
62
]
63
else:
64
raise ValueError(f"Invalid method {method}. \