A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix][Speculative Decoding] Extend Eagle quantization config fix to llama_eagle.py (#26590)
Signed-off-by: Rahul Tuli <rtuli@redhat.com>
Rahul Tuli committed
Oct 13, 2025 at 22:47 UTC
e3b90c1ba2e101ca99f5fa773b7851d1aa531ead
1 file changed
+12
vllm/model_executor/models/llama_eagle.py
+12
@@ -12,6 +12,7 @@ from vllm.config import VllmConfig
12
from vllm.distributed.parallel_state import get_pp_group
13
from vllm.logger import init_logger
14
from vllm.model_executor.layers.logits_processor import LogitsProcessor
15
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
16
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
17
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
@@ -37,6 +38,17 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
38
del self.input_layernorm
39
self.input_layernorm = nn.Identity()
40
41
+ def get_quant_config(self, vllm_config: VllmConfig) -> QuantizationConfig | None:
42
+ """Use drafter's quantization config instead of verifier's."""
43
+ draft_model_config = vllm_config.speculative_config.draft_model_config
44
+ draft_load_config = vllm_config.load_config
45
+
46
+ return (
47
+ VllmConfig.get_quantization_config(draft_model_config, draft_load_config)
48
+ if draft_model_config
49
+ else None
50
+ )
51
+
52
53
@support_torch_compile
54
class LlamaModel(nn.Module):