A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 21:00 UTC
[Spec Decode, BugFix] Propagate norm_before_fc from Eagle3 speculator (#38111)
Signed-off-by: Shubhra Pandit <shubhra.pandit@gmail.com>
Shubhra Pandit committed
Mar 28, 2026 at 20:42 UTC
5b8c30d62b754b575e043ce2fc0dcbf8a64f6306
1 file changed
+2
vllm/transformers_utils/configs/speculators/algos.py
+2
@@ -22,6 +22,7 @@ def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None:
22
- draft_vocab_size: Size of the draft model's vocabulary
23
- target_hidden_size: Hidden size of the target model
24
- norm_before_residual: Whether to apply norm before residual connection
25
+ - norm_before_fc: Whether to apply RMSNorm before the fc projection
26
- eagle_aux_hidden_state_layer_ids: List of layer indices from the base
27
model to use as auxiliary inputs for the Eagle3 drafter. These layers
28
provide intermediate hidden states that help the drafter make better
@@ -34,6 +35,7 @@ def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None:
35
pre_trained_config["norm_before_residual"] = config_dict.get(
36
"norm_before_residual", True
37
)
38
+ pre_trained_config["norm_before_fc"] = config_dict.get("norm_before_fc", False)
39
pre_trained_config["architectures"] = ["Eagle3LlamaForCausalLM"]
40
if config_dict.get("eagle_aux_hidden_state_layer_ids"):
41
pre_trained_config["eagle_aux_hidden_state_layer_ids"] = config_dict[