@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2118d0565c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix][SpecDecode] Adjust Eagle model architecture to align with intended design (#11672)

Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com>

Sungjae Lee committed Jan 11, 2025 at 13:49 UTC 2118d0565cb52c69e0bbacdcf48af9ecef39e528
1 file changed +24 -2
vllm/model_executor/models/eagle.py
+24 -2
@@ -17,14 +17,30 @@ from vllm.sequence import IntermediateTensors
17 from .utils import maybe_prefix
18
19
20 +class DummyInputLayerNorm(nn.Module):
21 +
22 + def forward(self, x):
23 + return x
24 +
25 +
26 +class DummyOutputNorm(nn.Module):
27 +
28 + def forward(self, x, residual):
29 + if residual is None:
30 + return x
31 + else:
32 + return x, residual
33 +
34 +
35 class EAGLE(nn.Module):
36 """This class implements the EAGLE draft model from the paper: https://arxiv.org/pdf/2401.15077
37 Reference implementation: https://github.com/SafeAILab/EAGLE
38
39 Differences from reference implementation:
40 1. In reference, LlamaDecoderLayer implementation doesn't have
26 - input_layernorm for 1st decoder layer (https://github.com/SafeAILab/EAGLE/blob/7d065d084443fbfd386f88839efd7193c12be869/eagle/model/cnets.py#L427)
27 - but we do as HF implementation also does.
41 + input_layernorm for 1st decoder layer (https://github.com/SafeAILab/EAGLE/blob/7d065d084443fbfd386f88839efd7193c12be869/eagle/model/cnets.py#L427).
42 + Following this approach, our implementation also disables
43 + the input_layernorm for the first decoder layer.
44 2. We allow any decoder layer to be used in EAGLE whereas in reference
45 decoder layer is fixed to be LlamaDecoderLayer.
46 3. We have an optional token_map which reduces draft vocab to most
@@ -46,10 +62,16 @@ class EAGLE(nn.Module):
62
63 self.model = model_cls(vllm_config=vllm_config,
64 prefix=maybe_prefix(prefix, "model"))
65 +
66 self.fc = nn.Linear(config.model.hidden_size * 2,
67 config.model.hidden_size,
68 bias=getattr(self.config, "eagle_fc_bias", False))
69
70 + # Modify layer normalization and residual connections as suggested
71 + # in the EAGLE framework: https://github.com/SafeAILab/EAGLE
72 + self.model.model.layers[0].input_layernorm = DummyInputLayerNorm()
73 + self.model.model.norm = DummyOutputNorm()
74 +
75 self.orig_vocab_size = config.vocab_size
76 self.truncated_vocab_size = config.truncated_vocab_size
77 self.unpadded_vocab_size = self.truncated_vocab_size