A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix][Speculative Decoding] Fix Eagle3 quantization config issue (#25883)
Signed-off-by: Rahul Tuli <rtuli@redhat.com>
Rahul Tuli committed
Sep 29, 2025 at 21:07 UTC
145ac73317e0a255cd22ab5f4ac346124800be41
3 files changed
+22
-2
tests/speculative_decoding/speculators/test_eagle3.py
+3
@@ -14,6 +14,9 @@ from vllm.model_executor.models.interfaces import supports_eagle3
14
pytest.param(
15
"nm-testing/Speculator-Qwen3-8B-Eagle3-converted-071-quantized",
16
id="qwen3-eagle3-speculator"),
17
+ pytest.param(
18
+ "nm-testing/Speculator-Qwen3-8B-Eagle3-converted-071-quantized-w4a16",
19
+ id="qwen3-eagle3-speculator-w4a16-verifier"),
20
])
21
def test_eagle3_speculators_model(vllm_runner, example_prompts, model_path,
22
monkeypatch):
vllm/model_executor/models/llama.py
+6
-1
@@ -248,7 +248,7 @@ class LlamaDecoderLayer(nn.Module):
248
249
config = config or vllm_config.model_config.hf_config
250
cache_config = vllm_config.cache_config
251
- quant_config = vllm_config.quant_config
251
+ quant_config = self.get_quant_config(vllm_config)
252
253
self.hidden_size = config.hidden_size
254
rope_theta = getattr(config, "rope_theta", 10000)
@@ -328,6 +328,11 @@ class LlamaDecoderLayer(nn.Module):
328
hidden_states = self.mlp(hidden_states)
329
return hidden_states, residual
330
331
+ def get_quant_config(
332
+ self, vllm_config: VllmConfig) -> Optional[QuantizationConfig]:
333
+ """Get quantization config for this layer. Override in subclasses."""
334
+ return vllm_config.quant_config
335
+
336
337
@support_torch_compile
338
class LlamaModel(nn.Module):
vllm/model_executor/models/llama_eagle3.py
+13
-1
@@ -13,6 +13,8 @@ from vllm.logger import init_logger
13
from vllm.model_executor.layers.layernorm import RMSNorm
14
from vllm.model_executor.layers.linear import QKVParallelLinear
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
+from vllm.model_executor.layers.quantization.base_config import (
17
+ QuantizationConfig)
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
20
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -33,7 +35,7 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
35
super().__init__(vllm_config, prefix=prefix, config=config)
36
37
config = config or vllm_config.model_config.hf_config
36
- quant_config = vllm_config.quant_config
38
+ quant_config = self.get_quant_config(vllm_config)
39
40
# override qkv
41
self.self_attn.qkv_proj = QKVParallelLinear(
@@ -53,6 +55,16 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
55
else:
56
self._residual_norm = self._norm_after_residual
57
58
+ def get_quant_config(
59
+ self, vllm_config: VllmConfig) -> Optional[QuantizationConfig]:
60
+ """Use drafter's quantization config instead of verifier's."""
61
+ draft_model_config = vllm_config.speculative_config.draft_model_config
62
+ draft_load_config = vllm_config.load_config
63
+
64
+ return VllmConfig.get_quantization_config(
65
+ draft_model_config,
66
+ draft_load_config) if draft_model_config else None
67
+
68
def _norm_before_residual(
69
self,
70
hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: