A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 12:30 UTC
[Misc][Spec Decode] support different load config for draft model (#34022)
Signed-off-by: zzhengkai <zzhengkai@devgpu049.ldc1.facebook.com> Co-authored-by: zzhengkai <zzhengkai@devgpu049.ldc1.facebook.com>
Zhengkai Zhang committed
Feb 10, 2026 at 14:52 UTC
6f2f59f2b333151aac19f8ca7bf71d83c1a7c068
3 files changed
+8
-1
vllm/config/speculative.py
+5
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal, get_args
7
from pydantic import Field, SkipValidation, model_validator
8
from typing_extensions import Self
9
10
+from vllm.config import LoadConfig
11
from vllm.config.model import ModelConfig
12
from vllm.config.parallel import ParallelConfig
13
from vllm.config.utils import config
@@ -160,6 +161,10 @@ class SpeculativeConfig:
161
tokens with estimated probability (based on frequency counts) greater than
162
or equal to this value."""
163
164
+ draft_load_config: LoadConfig | None = None
165
+ """Load config for the draft model. If not specified, will use the load
166
+ config from the target model."""
167
+
168
def compute_hash(self) -> str:
169
"""
170
WARNING: Whenever a new field is added to this config,
vllm/model_executor/model_loader/__init__.py
+2
-1
@@ -128,8 +128,9 @@ def get_model(
128
vllm_config: VllmConfig,
129
model_config: ModelConfig | None = None,
130
prefix: str = "",
131
+ load_config: LoadConfig | None = None,
132
) -> nn.Module:
132
- loader = get_model_loader(vllm_config.load_config)
133
+ loader = get_model_loader(load_config or vllm_config.load_config)
134
if model_config is None:
135
model_config = vllm_config.model_config
136
return loader.load_model(
vllm/v1/spec_decode/eagle.py
+1
@@ -1286,6 +1286,7 @@ class SpecDecodeBaseProposer:
1286
model = get_model(
1287
vllm_config=self.vllm_config,
1288
model_config=self.speculative_config.draft_model_config,
1289
+ load_config=self.speculative_config.draft_load_config,
1290
)
1291
return model
1292