@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80ea3af1a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[CI][Spec Decode] fix: broken test for EAGLE model (#11972)

Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com>

Sungjae Lee committed Jan 13, 2025 at 15:50 UTC 80ea3af1a06ff445e6cdede072bda1429c9dac06
2 files changed +13 -2
.buildkite/test-pipeline.yaml
+3 -1
@@ -231,13 +231,15 @@ steps:
231 - pytest -v -s test_logits_processor.py
232 - pytest -v -s model_executor/test_guided_processors.py
233
234 -- label: Speculative decoding tests # 30min
234 +- label: Speculative decoding tests # 40min
235 source_file_dependencies:
236 - vllm/spec_decode
237 - tests/spec_decode
238 + - vllm/model_executor/models/eagle.py
239 commands:
240 - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
241 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
242 + - pytest -v -s spec_decode/e2e/test_eagle_correctness.py
243
244 - label: LoRA Test %N # 15min each
245 mirror_hardwares: [amd]
vllm/model_executor/models/eagle.py
+10 -1
@@ -19,6 +19,11 @@ from .utils import maybe_prefix
19
20 class DummyInputLayerNorm(nn.Module):
21
22 + def __init__(self, weight=None, bias=None):
23 + super().__init__()
24 + self.weight = nn.Parameter(weight) if weight is not None else None
25 + self.bias = nn.Parameter(bias) if bias is not None else None
26 +
27 def forward(self, x):
28 return x
29
@@ -69,7 +74,11 @@ class EAGLE(nn.Module):
74
75 # Modify layer normalization and residual connections as suggested
76 # in the EAGLE framework: https://github.com/SafeAILab/EAGLE
72 - self.model.model.layers[0].input_layernorm = DummyInputLayerNorm()
77 + # While weights and biases are generally not needed,
78 + # they are retained here to support certain unit tests
79 + # (e.g., spec_decode/e2e/test_eagle_correctness.py).
80 + self.model.model.layers[0].input_layernorm = DummyInputLayerNorm(
81 + weight=self.model.model.layers[0].input_layernorm.weight)
82 self.model.model.norm = DummyOutputNorm()
83
84 self.orig_vocab_size = config.vocab_size