A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix][SpecDecode] Adjust Eagle model architecture to align with intended design (#11672)
Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com>
Sungjae Lee committed
Jan 11, 2025 at 13:49 UTC
2118d0565cb52c69e0bbacdcf48af9ecef39e528
1 file changed
+24
-2
vllm/model_executor/models/eagle.py
+24
-2
@@ -17,14 +17,30 @@ from vllm.sequence import IntermediateTensors
17
from .utils import maybe_prefix
18
19
20
+class DummyInputLayerNorm(nn.Module):
21
+
22
+ def forward(self, x):
23
+ return x
24
+
25
+
26
+class DummyOutputNorm(nn.Module):
27
+
28
+ def forward(self, x, residual):
29
+ if residual is None:
30
+ return x
31
+ else:
32
+ return x, residual
33
+
34
+
35
class EAGLE(nn.Module):
36
"""This class implements the EAGLE draft model from the paper: https://arxiv.org/pdf/2401.15077
37
Reference implementation: https://github.com/SafeAILab/EAGLE
38
39
Differences from reference implementation:
40
1. In reference, LlamaDecoderLayer implementation doesn't have
26
- input_layernorm for 1st decoder layer (https://github.com/SafeAILab/EAGLE/blob/7d065d084443fbfd386f88839efd7193c12be869/eagle/model/cnets.py#L427)
27
- but we do as HF implementation also does.
41
+ input_layernorm for 1st decoder layer (https://github.com/SafeAILab/EAGLE/blob/7d065d084443fbfd386f88839efd7193c12be869/eagle/model/cnets.py#L427).
42
+ Following this approach, our implementation also disables
43
+ the input_layernorm for the first decoder layer.
44
2. We allow any decoder layer to be used in EAGLE whereas in reference
45
decoder layer is fixed to be LlamaDecoderLayer.
46
3. We have an optional token_map which reduces draft vocab to most
@@ -46,10 +62,16 @@ class EAGLE(nn.Module):
62
63
self.model = model_cls(vllm_config=vllm_config,
64
prefix=maybe_prefix(prefix, "model"))
65
+
66
self.fc = nn.Linear(config.model.hidden_size * 2,
67
config.model.hidden_size,
68
bias=getattr(self.config, "eagle_fc_bias", False))
69
70
+ # Modify layer normalization and residual connections as suggested
71
+ # in the EAGLE framework: https://github.com/SafeAILab/EAGLE
72
+ self.model.model.layers[0].input_layernorm = DummyInputLayerNorm()
73
+ self.model.model.norm = DummyOutputNorm()
74
+
75
self.orig_vocab_size = config.vocab_size
76
self.truncated_vocab_size = config.truncated_vocab_size
77
self.unpadded_vocab_size = self.truncated_vocab_size