A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Fix][Spec Decode] Fix llama4 draft loading with different quantization (#27136)
Signed-off-by: linzebing <linzebing1995@gmail.com>
Zebing Lin committed
Oct 21, 2025 at 02:19 UTC
be4445072c4e1e5e3a2ebf0552e432fc86f137ca
1 file changed
+17
-10
vllm/model_executor/models/llama4_eagle.py
+17
-10
@@ -60,16 +60,23 @@ class LlamaModel(nn.Module):
60
prefix=maybe_prefix(prefix, "embed_tokens"),
61
)
62
63
- self.layers = nn.ModuleList(
64
- [
65
- Llama4DecoderLayer(
66
- vllm_config=vllm_config,
67
- prefix=maybe_prefix(prefix, f"layers.{i + start_layer_id}"),
68
- config=self.config,
69
- )
70
- for i in range(self.config.num_hidden_layers)
71
- ]
72
- )
63
+ # Temporarily modify vllm_config.quant_config for draft model layers
64
+ original_quant_config = vllm_config.quant_config
65
+ vllm_config.quant_config = quant_config
66
+ try:
67
+ self.layers = nn.ModuleList(
68
+ [
69
+ Llama4DecoderLayer(
70
+ vllm_config=vllm_config,
71
+ prefix=maybe_prefix(prefix, f"layers.{i + start_layer_id}"),
72
+ config=self.config,
73
+ )
74
+ for i in range(self.config.num_hidden_layers)
75
+ ]
76
+ )
77
+ finally:
78
+ # Restore original quant_config
79
+ vllm_config.quant_config = original_quant_config
80
self.fc = torch.nn.Linear(
81
self.config.hidden_size * 2, self.config.hidden_size, bias=False
82
)