A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Speculative Decoding] EAGLE Implementation with Top-1 proposer (#6830)
Abhinav Goyal committed
Aug 22, 2024 at 15:12 UTC
a3fce56b887cba61b8e05d50f01be4309ab2c42c
17 files changed
+854
-83
tests/spec_decode/e2e/test_eagle_correctness.py
new
+268
@@ -0,0 +1,268 @@
1
+"""This docstring details important information on the testing methodology.
2
+
3
+Most of the tests rely on "greedy equality", where we expect the output of
4
+speculative decoding on a sequence to exactly match the output of normal non-
5
+speculative decoding.
6
+
7
+Since speculative decoding with rejection sampling guarantees that the output
8
+distribution matches the target model's output distribution (up to hardware
9
+numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
10
+equality.
11
+
12
+However, we still need to verify below scenario could be passed:
13
+ * Batch size 1 greedy equality
14
+ * Batch size >1 greedy equality
15
+ * Test greedy equality under preemption
16
+ * Test greedy equality under various number of speculative tokens.
17
+
18
+With those tests, we can say at least, EAGLE would not break the
19
+correctess for the target model outputs.
20
+"""
21
+
22
+import pytest
23
+
24
+from .conftest import run_greedy_equality_correctness_test
25
+
26
+# main model
27
+MAIN_MODEL = "JackFram/llama-68m"
28
+
29
+# speculative model
30
+SPEC_MODEL = "abhigoyal/vllm-eagle-llama-68m-random"
31
+
32
+# max. number of speculative tokens: this corresponds to
33
+# num_heads in the config.json of the speculator model.
34
+MAX_SPEC_TOKENS = 4
35
+
36
+# precision
37
+PRECISION = "float32"
38
+
39
+
40
+@pytest.mark.parametrize(
41
+ "common_llm_kwargs",
42
+ [{
43
+ # Skip cuda graph recording for fast test.
44
+ "enforce_eager": True,
45
+
46
+ # Required for spec decode.
47
+ "use_v2_block_manager": True,
48
+
49
+ # Print spec metrics.
50
+ "disable_log_stats": False,
51
+
52
+ # Precision
53
+ "dtype": PRECISION,
54
+
55
+ # Main model
56
+ "model": MAIN_MODEL,
57
+ }])
58
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
59
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
60
+@pytest.mark.parametrize("test_llm_kwargs", [
61
+ {
62
+ "speculative_model": SPEC_MODEL,
63
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
64
+ },
65
+])
66
+@pytest.mark.parametrize("output_len", [
67
+ 128,
68
+])
69
+@pytest.mark.parametrize("batch_size", [1, 32])
70
+@pytest.mark.parametrize("seed", [1])
71
+def test_eagle_e2e_greedy_correctness(baseline_llm_generator,
72
+ test_llm_generator, batch_size: int,
73
+ output_len: int):
74
+ """Verify greedy equality with different batch size."""
75
+ run_greedy_equality_correctness_test(baseline_llm_generator,
76
+ test_llm_generator,
77
+ batch_size,
78
+ max_output_len=output_len,
79
+ force_output_len=True)
80
+
81
+
82
+@pytest.mark.parametrize(
83
+ "common_llm_kwargs",
84
+ [{
85
+ "enforce_eager": False,
86
+
87
+ # Required for spec decode.
88
+ "use_v2_block_manager": True,
89
+
90
+ # Print spec metrics.
91
+ "disable_log_stats": False,
92
+
93
+ # Precision
94
+ "dtype": PRECISION,
95
+
96
+ # Main model
97
+ "model": MAIN_MODEL,
98
+ }])
99
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
100
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
101
+@pytest.mark.parametrize("test_llm_kwargs", [
102
+ {
103
+ "speculative_model": SPEC_MODEL,
104
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
105
+ },
106
+])
107
+@pytest.mark.parametrize("output_len", [
108
+ 128,
109
+])
110
+@pytest.mark.parametrize("batch_size", [1, 32])
111
+@pytest.mark.parametrize("seed", [1])
112
+def test_eagle_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
113
+ test_llm_generator,
114
+ batch_size: int,
115
+ output_len: int):
116
+ """Verify greedy equality with cuda graph enabled and different
117
+ batch sizes."""
118
+ run_greedy_equality_correctness_test(baseline_llm_generator,
119
+ test_llm_generator,
120
+ batch_size,
121
+ max_output_len=output_len,
122
+ force_output_len=True)
123
+
124
+
125
+@pytest.mark.parametrize(
126
+ "common_llm_kwargs",
127
+ [{
128
+ "block_size": 8,
129
+ # 2 for small prompt, 256//8 for generated.
130
+ "num_gpu_blocks_override": 2 + 256 // 8,
131
+ "max_model_len": (2 + 256 // 8) * 8,
132
+
133
+ # Skip cuda graph recording for fast test.
134
+ "enforce_eager": True,
135
+
136
+ # Required for spec decode.
137
+ "use_v2_block_manager": True,
138
+
139
+ # Precision
140
+ "dtype": PRECISION,
141
+
142
+ # Main model
143
+ "model": MAIN_MODEL,
144
+ }])
145
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
146
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
147
+@pytest.mark.parametrize("test_llm_kwargs", [
148
+ {
149
+ "speculative_model": SPEC_MODEL,
150
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
151
+ },
152
+])
153
+@pytest.mark.parametrize(
154
+ "output_len",
155
+ [
156
+ # Use small output len for fast test.
157
+ 128,
158
+ ])
159
+@pytest.mark.parametrize("batch_size", [4])
160
+@pytest.mark.parametrize("seed", [1])
161
+def test_eagle_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
162
+ test_llm_generator,
163
+ batch_size: int,
164
+ output_len: int):
165
+ """Verify greedy equality, even when some sequences are preempted mid-
166
+ generation.
167
+ """
168
+ run_greedy_equality_correctness_test(baseline_llm_generator,
169
+ test_llm_generator,
170
+ batch_size,
171
+ max_output_len=output_len,
172
+ force_output_len=True)
173
+
174
+
175
+@pytest.mark.parametrize(
176
+ "common_llm_kwargs",
177
+ [{
178
+ # Skip cuda graph recording for fast test.
179
+ "enforce_eager": True,
180
+
181
+ # Required for spec decode.
182
+ "use_v2_block_manager": True,
183
+
184
+ # Precision
185
+ "dtype": PRECISION,
186
+
187
+ # Main model
188
+ "model": MAIN_MODEL,
189
+ }])
190
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
191
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
192
+@pytest.mark.parametrize(
193
+ "test_llm_kwargs",
194
+ [
195
+ {
196
+ "speculative_model": SPEC_MODEL,
197
+ "num_speculative_tokens": k,
198
+ }
199
+ # Try a range of num. speculative tokens
200
+ for k in range(1, 1 + MAX_SPEC_TOKENS)
201
+ ])
202
+@pytest.mark.parametrize("batch_size", [2])
203
+@pytest.mark.parametrize(
204
+ "output_len",
205
+ [
206
+ # Use smaller output len for fast test.
207
+ 32,
208
+ ])
209
+@pytest.mark.parametrize("seed", [1])
210
+def test_eagle_different_k(baseline_llm_generator, test_llm_generator,
211
+ batch_size: int, output_len: int):
212
+ """Verify that eagle speculative decoding produces exact equality
213
+ to without spec decode with different values of num_speculative_tokens.
214
+ """
215
+ run_greedy_equality_correctness_test(baseline_llm_generator,
216
+ test_llm_generator,
217
+ batch_size,
218
+ max_output_len=output_len,
219
+ force_output_len=True)
220
+
221
+
222
+@pytest.mark.parametrize(
223
+ "common_llm_kwargs",
224
+ [{
225
+ # Skip cuda graph recording for fast test.
226
+ "enforce_eager": True,
227
+
228
+ # Required for spec decode.
229
+ "use_v2_block_manager": True,
230
+
231
+ # Precision
232
+ "dtype": PRECISION,
233
+
234
+ # Main model
235
+ "model": MAIN_MODEL,
236
+ }])
237
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
238
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
239
+@pytest.mark.parametrize("test_llm_kwargs",
240
+ [{
241
+ "speculative_model": SPEC_MODEL,
242
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
243
+ "speculative_disable_by_batch_size": 4
244
+ }])
245
+@pytest.mark.parametrize("batch_size", [1, 5])
246
+@pytest.mark.parametrize(
247
+ "output_len",
248
+ [
249
+ # Use smaller output len for fast test.
250
+ 32,
251
+ ])
252
+@pytest.mark.parametrize("seed", [1])
253
+def test_eagle_disable_queue(baseline_llm_generator, test_llm_generator,
254
+ batch_size: int, output_len: int):
255
+ """Verify that eagle speculative decoding produces exact equality
256
+ to without spec decode when speculation is disabled for large
257
+ batch sizes.
258
+ """
259
+ run_greedy_equality_correctness_test(baseline_llm_generator,
260
+ test_llm_generator,
261
+ batch_size,
262
+ max_output_len=output_len,
263
+ force_output_len=True)
264
+
265
+
266
+if __name__ == "__main__":
267
+ import pytest
268
+ pytest.main([__file__])
tests/spec_decode/e2e/test_medusa_correctness.py
+56
-12
@@ -70,8 +70,9 @@ PRECISION = "float32"
70
])
71
@pytest.mark.parametrize("batch_size", [1, 32])
72
@pytest.mark.parametrize("seed", [1])
73
-def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
74
- batch_size: int, output_len: int):
73
+def test_medusa_e2e_greedy_correctness(baseline_llm_generator,
74
+ test_llm_generator, batch_size: int,
75
+ output_len: int):
76
"""Verify greedy equality with different batch size."""
77
run_greedy_equality_correctness_test(baseline_llm_generator,
78
test_llm_generator,
@@ -80,6 +81,49 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
81
force_output_len=True)
82
83
84
+@pytest.mark.parametrize(
85
+ "common_llm_kwargs",
86
+ [{
87
+ "enforce_eager": False,
88
+
89
+ # Required for spec decode.
90
+ "use_v2_block_manager": True,
91
+
92
+ # Print spec metrics.
93
+ "disable_log_stats": False,
94
+
95
+ # Precision
96
+ "dtype": PRECISION,
97
+
98
+ # Main model
99
+ "model": MAIN_MODEL,
100
+ }])
101
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
102
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
103
+@pytest.mark.parametrize("test_llm_kwargs", [
104
+ {
105
+ "speculative_model": SPEC_MODEL,
106
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
107
+ },
108
+])
109
+@pytest.mark.parametrize("output_len", [
110
+ 128,
111
+])
112
+@pytest.mark.parametrize("batch_size", [1, 32])
113
+@pytest.mark.parametrize("seed", [1])
114
+def test_medusa_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
115
+ test_llm_generator,
116
+ batch_size: int,
117
+ output_len: int):
118
+ """Verify greedy equality with cuda graph enabled and different
119
+ batch sizes."""
120
+ run_greedy_equality_correctness_test(baseline_llm_generator,
121
+ test_llm_generator,
122
+ batch_size,
123
+ max_output_len=output_len,
124
+ force_output_len=True)
125
+
126
+
127
@pytest.mark.parametrize(
128
"common_llm_kwargs",
129
[{
@@ -116,10 +160,10 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
160
])
161
@pytest.mark.parametrize("batch_size", [4])
162
@pytest.mark.parametrize("seed", [1])
119
-def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
120
- test_llm_generator,
121
- batch_size: int,
122
- output_len: int):
163
+def test_medusa_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
164
+ test_llm_generator,
165
+ batch_size: int,
166
+ output_len: int):
167
"""Verify greedy equality, even when some sequences are preempted mid-
168
generation.
169
"""
@@ -165,9 +209,9 @@ def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
209
32,
210
])
211
@pytest.mark.parametrize("seed", [1])
168
-def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
169
- batch_size: int, output_len: int):
170
- """Verify that mlp speculative decoding produces exact equality
212
+def test_medusa_different_k(baseline_llm_generator, test_llm_generator,
213
+ batch_size: int, output_len: int):
214
+ """Verify that medusa speculative decoding produces exact equality
215
to without spec decode with different values of num_speculative_tokens.
216
"""
217
run_greedy_equality_correctness_test(baseline_llm_generator,
@@ -208,9 +252,9 @@ def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
252
32,
253
])
254
@pytest.mark.parametrize("seed", [1])
211
-def test_mlp_disable_queue(baseline_llm_generator, test_llm_generator,
212
- batch_size: int, output_len: int):
213
- """Verify that mlp speculative decoding produces exact equality
255
+def test_medusa_disable_queue(baseline_llm_generator, test_llm_generator,
256
+ batch_size: int, output_len: int):
257
+ """Verify that medusa speculative decoding produces exact equality
258
to without spec decode when speculation is disabled for large
259
batch sizes.
260
"""
tests/spec_decode/test_multi_step_worker.py
+35
-1
@@ -6,7 +6,8 @@ import pytest
6
import torch
7
8
from vllm.model_executor.utils import set_random_seed
9
-from vllm.sequence import ExecuteModelRequest, Logprob, SamplerOutput
9
+from vllm.sequence import (ExecuteModelRequest, HiddenStates, Logprob,
10
+ SamplerOutput, get_all_seq_ids)
11
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
12
from vllm.spec_decode.multi_step_worker import MultiStepWorker
13
from vllm.spec_decode.top1_proposer import Top1Proposer
@@ -690,3 +691,36 @@ def test_use_draft_model_runner_advance_step():
691
worker.execute_model(execute_model_req=execute_model_req)
692
call_args_list = worker.model_runner._gpu_advance_step.call_args_list
693
assert len(call_args_list) == 1
694
+
695
+
696
+@torch.inference_mode()
697
+def test_expand_execute_model_request_sync_with_expand_hidden_states():
698
+ """
699
+ In this test we verify that the logic for expanding the
700
+ seq_group_metadata_list remains in sync with the expansion logic of
701
+ the HiddenStates in _expand_execute_model_request.
702
+ """
703
+ k = 5
704
+ batch_size = 16
705
+ seq_with_bonus_token_in_last_step = [1, 3, 8, 10, 13, 15]
706
+
707
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
708
+
709
+ execute_model_request = ExecuteModelRequest(
710
+ seq_group_metadata_list,
711
+ previous_hidden_states=HiddenStates(
712
+ torch.arange(batch_size), seq_group_metadata_list,
713
+ torch.arange(batch_size, 2 * batch_size)))
714
+
715
+ expanded_execute_model_request, orig_seq_group_ids = MultiStepWorker.\
716
+ _expand_execute_model_request(execute_model_request,
717
+ seq_with_bonus_token_in_last_step)
718
+
719
+ all_seq_ids = torch.tensor(
720
+ get_all_seq_ids(
721
+ expanded_execute_model_request.seq_group_metadata_list))
722
+ ref_expanded_hidden_states = all_seq_ids + batch_size
723
+ ref_expanded_hidden_states[orig_seq_group_ids] -= batch_size
724
+
725
+ assert (ref_expanded_hidden_states == expanded_execute_model_request.
726
+ previous_hidden_states.hidden_states).all().item()
vllm/model_executor/models/__init__.py
+1
@@ -60,6 +60,7 @@ _GENERATION_MODELS = {
60
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
61
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
62
"MedusaModel": ("medusa", "Medusa"),
63
+ "EAGLEModel": ("eagle", "EAGLE"),
64
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
65
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
66
}
vllm/model_executor/models/eagle.py
new
+161
@@ -0,0 +1,161 @@
1
+from typing import Iterable, List, Optional, Tuple
2
+
3
+import torch
4
+import torch.nn as nn
5
+
6
+from vllm.attention.backends.abstract import AttentionMetadata
7
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
8
+from vllm.model_executor.layers.vocab_parallel_embedding import (
9
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
10
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
11
+from vllm.model_executor.models import ModelRegistry
12
+from vllm.model_executor.sampling_metadata import SamplingMetadata
13
+from vllm.sequence import IntermediateTensors, SamplerOutput
14
+from vllm.transformers_utils.configs.eagle import EAGLEConfig
15
+
16
+
17
+class EAGLE(nn.Module):
18
+ """This class implements the EAGLE draft model from the paper: https://arxiv.org/pdf/2401.15077
19
+ Reference implementation: https://github.com/SafeAILab/EAGLE
20
+
21
+ Differences from reference implementation:
22
+ 1. In reference, LlamaDecoderLayer implementation doesn't have
23
+ input_layernorm for 1st decoder layer (https://github.com/SafeAILab/EAGLE/blob/7d065d084443fbfd386f88839efd7193c12be869/eagle/model/cnets.py#L427)
24
+ but we do as HF implementation also does.
25
+ 2. We allow any decoder layer to be used in EAGLE whereas in reference
26
+ decoder layer is fixed to be LlamaDecoderLayer.
27
+ 3. We have an optional token_map which reduces draft vocab to most
28
+ frequently used tokens to give some additional speed-up by reducing
29
+ sampling overhead. This is disabled unless the checkpoint file has
30
+ explicit token_map tensor and config has an optional attribute
31
+ truncated_vocab_size < vocab_size. To use this technique, one has to find
32
+ the top-k most frequent tokens in target dataset and add that as a tensor
33
+ in the draft checkpoint (using key token_map). Also, the draft config
34
+ needs to have truncated_vocab_size (=k) as an attribute."""
35
+
36
+ def __init__(self, config: EAGLEConfig, *args, **kwargs) -> None:
37
+ super().__init__()
38
+ self.config = config
39
+
40
+ architectures = getattr(self.config.model, "architectures", [])
41
+ model_cls, _ = ModelRegistry.resolve_model_cls(architectures)
42
+
43
+ self.model = model_cls(self.config.model, *args, **kwargs)
44
+ self.fc = nn.Linear(config.model.hidden_size * 2,
45
+ config.model.hidden_size,
46
+ bias=False)
47
+
48
+ self.orig_vocab_size = config.vocab_size
49
+ self.truncated_vocab_size = config.truncated_vocab_size
50
+ self.unpadded_vocab_size = self.truncated_vocab_size
51
+
52
+ self.lm_head = ParallelLMHead(
53
+ self.unpadded_vocab_size,
54
+ config.hidden_size,
55
+ org_num_embeddings=self.truncated_vocab_size,
56
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE,
57
+ )
58
+
59
+ logit_scale = getattr(config, "logit_scale", 1.0)
60
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
61
+ self.truncated_vocab_size,
62
+ logit_scale)
63
+
64
+ # Token map is a idx to token mapping to reduce the vocab size for
65
+ # the draft model. Using smaller vocab size for draft, containing
66
+ # only most frequent tokens reduces the speculation overhead. This
67
+ # doesn't affect the acceptance rate much and thus gives more speed
68
+ # -up. By default, this is disabled and is only used if the EAGLE
69
+ # checkpoint file has token_map tensor.
70
+ self.token_map = None
71
+
72
+ @property
73
+ def sampler(self):
74
+ return self.model.sampler
75
+
76
+ def forward(
77
+ self,
78
+ input_ids: torch.Tensor,
79
+ positions: torch.Tensor,
80
+ kv_caches: List[torch.Tensor],
81
+ attn_metadata: AttentionMetadata,
82
+ previous_hidden_states: torch.Tensor,
83
+ intermediate_tensors: Optional[IntermediateTensors] = None,
84
+ ) -> torch.Tensor:
85
+
86
+ tok_embeds = self.model.model.embed_tokens(input_ids)
87
+ inputs_embeds = self.fc(
88
+ torch.cat([tok_embeds, previous_hidden_states], dim=-1))
89
+
90
+ inputs_embeds[positions == 0] = 0 # masking inputs at position=0
91
+
92
+ hidden_states = self.model.model(
93
+ input_ids=None,
94
+ inputs_embeds=inputs_embeds,
95
+ positions=positions,
96
+ kv_caches=kv_caches,
97
+ attn_metadata=attn_metadata,
98
+ intermediate_tensors=intermediate_tensors)
99
+ return hidden_states
100
+
101
+ def compute_logits(self, hidden_states: torch.Tensor,
102
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
103
+ logits = self.logits_processor(self.lm_head, hidden_states,
104
+ sampling_metadata)
105
+
106
+ if self.token_map is not None:
107
+ _logits = logits
108
+ logits = -torch.inf * torch.ones(
109
+ size=(*_logits.shape[:-1], self.orig_vocab_size),
110
+ device=_logits.device,
111
+ dtype=_logits.dtype)
112
+
113
+ logits[..., self.token_map] = _logits
114
+
115
+ return logits
116
+
117
+ def sample(
118
+ self,
119
+ logits: torch.Tensor,
120
+ sampling_metadata: SamplingMetadata,
121
+ ) -> Optional[SamplerOutput]:
122
+ next_tokens = self.sampler(logits, sampling_metadata)
123
+ return next_tokens
124
+
125
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
126
+ # This implementation is incompitable with https://huggingface.co/yuhuili/EAGLE-LLaMA3-Instruct-8B
127
+ # due to missing lm_head weights and its config being that of a
128
+ # Llama model. Here's a compatible version with the same weights:
129
+ # https://huggingface.co/abhigoyal/EAGLE-LLaMA3-Instruct-8B-vllm
130
+ # Also, here's an example script for converting trained EAGLE
131
+ # checkpoint to vLLM compatible version: https://gist.github.com/abhigoyal1997/1e7a4109ccb7704fbc67f625e86b2d6d
132
+ model_weights = {}
133
+ for name, loaded_weight in weights:
134
+ if name == "token_map":
135
+ if self.config.truncated_vocab_size < self.config.vocab_size:
136
+ self.token_map = nn.Parameter(loaded_weight,
137
+ requires_grad=False)
138
+ elif name.startswith("fc."):
139
+ weight_loader = getattr(self.fc.weight, "weight_loader",
140
+ default_weight_loader)
141
+ weight_loader(self.fc.weight, loaded_weight)
142
+ elif name.startswith("model.lm_head.") or name.startswith(
143
+ "model.model."):
144
+ model_weights[name.split("model.", 1)[-1]] = loaded_weight
145
+ elif name.startswith("lm_head.") or name.startswith("model."):
146
+ model_weights[name] = loaded_weight
147
+ else:
148
+ model_weights[f"model.{name}"] = loaded_weight
149
+
150
+ lm_head_weight = model_weights.pop("lm_head.weight")
151
+
152
+ if self.token_map is not None and\
153
+ lm_head_weight.shape[0] > self.token_map.shape[0]:
154
+
155
+ lm_head_weight = lm_head_weight[self.token_map]
156
+
157
+ weight_loader = getattr(self.lm_head.weight, "weight_loader",
158
+ default_weight_loader)
159
+ weight_loader(self.lm_head.weight, lm_head_weight)
160
+
161
+ self.model.load_weights(model_weights.items())
vllm/model_executor/models/medusa.py
+19
@@ -30,6 +30,19 @@ class ResidualBlock(nn.Module):
30
31
32
class Medusa(nn.Module):
33
+ """This class implements the Medusa draft model from the paper: https://arxiv.org/abs/2401.10774
34
+ Reference implementation: https://github.com/FasterDecoding/Medusa
35
+
36
+ Differences from reference implementation:
37
+ 1. Currently this only supports generating proposals from top-1 tokens.
38
+ 2. We have an optional token_map which reduces draft vocab to most
39
+ frequently used tokens to give some additional speed-up by reducing
40
+ sampling overhead. This is disabled unless the checkpoint file has
41
+ explicit token_map tensor and config has an optional attribute
42
+ truncated_vocab_size < vocab_size. To use this technique, one has to find
43
+ the top-k most frequent tokens in target dataset and add that as a tensor
44
+ in the draft checkpoint (using key token_map). Also, the draft config
45
+ needs to have truncated_vocab_size (=k) as an attribute."""
46
47
def __init__(self, config: MedusaConfig, **_) -> None:
48
super().__init__()
@@ -57,6 +70,12 @@ class Medusa(nn.Module):
70
self.truncated_vocab_size,
71
logit_scale)
72
73
+ # Token map is a idx to token mapping to reduce the vocab size for
74
+ # the draft model. Using smaller vocab size for draft, containing
75
+ # only most frequent tokens reduces the speculation overhead. This
76
+ # doesn't affect the acceptance rate much and thus gives more speed
77
+ # -up. By default, this is disabled and is only used if the EAGLE
78
+ # checkpoint file has token_map tensor.
79
self.token_map = None
80
81
def forward(self, hidden_states: torch.Tensor) -> List[torch.Tensor]:
vllm/sequence.py
+60
-9
@@ -1092,6 +1092,10 @@ class SamplerOutput(
1092
# Optional last hidden states from the model.
1093
hidden_states: Optional[torch.Tensor] = None
1094
1095
+ # Optional prefill hidden states from the model
1096
+ # (used for models like EAGLE).
1097
+ prefill_hidden_states: Optional[torch.Tensor] = None
1098
+
1099
# Time taken in the forward pass for this across all workers
1100
model_forward_time: Optional[float] = None
1101
@@ -1176,40 +1180,87 @@ class HiddenStates(msgspec.Struct, array_like=True,
1180
omit_defaults=True): # type: ignore[call-arg]
1181
"""Hidden states corresponding to in-progress sequences.
1182
Used in speculative decoding to pass hidden states from
1179
- the target model to the proposer model in the subsequent step.
1183
+ the target model to the proposer model.
1184
1185
seq_ids are the sequence ids of each entry of the batch
1186
dimension of the hidden_states tensor"""
1183
-
1184
- seq_group_metadata_list: List[SequenceGroupMetadata]
1187
+ # Scorer hidden states. For prefill step, it is used for hidden states of
1188
+ # all tokens, whereas for decode step, it use used for last accepted tokens.
1189
hidden_states: torch.Tensor
1190
+ # The sequence group metadata list. Only needed for decode step.
1191
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
1192
+ # Scorer hidden states of the 2nd last token proposed by the proposer (
1193
+ # irrespective of whether it was accepted or not). Only used for cases when
1194
+ # last proposed token is accepted (i.e., in case of bonus tokens). For the
1195
+ # case of no bonus tokens, these are ignored.
1196
+ second_last_token_hidden_states: Optional[torch.Tensor] = None
1197
+
1198
_seq_ids: List[int] = msgspec.field(default_factory=list)
1199
1200
def __post_init__(self):
1189
- self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list)
1190
- assert len(self.seq_group_metadata_list) == len(self.hidden_states)
1201
+ if self.seq_group_metadata_list is not None:
1202
+ assert len(self.seq_group_metadata_list) == len(self.hidden_states)
1203
+ self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list)
1204
1205
@property
1206
def seq_ids(self) -> List[int]:
1207
return self._seq_ids
1208
1196
- def update(self, seq_group_metadata_list: List[SequenceGroupMetadata],
1197
- hidden_states: torch.Tensor) -> None:
1198
- """Update hidden states from target model invocation."""
1209
+ def update(self,
1210
+ hidden_states: torch.Tensor,
1211
+ seq_group_metadata_list: List[SequenceGroupMetadata],
1212
+ second_last_token_hidden_states: Optional[torch.Tensor] = None):
1213
+ """Update hidden states from target model invocation. Only used for
1214
+ decode steps"""
1215
assert len(seq_group_metadata_list) == len(hidden_states)
1216
self._seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
1217
self.hidden_states = torch.cat([self.hidden_states, hidden_states])
1218
1219
+ if self.second_last_token_hidden_states is not None:
1220
+ # Adding dummy hidden_states to this to maintain same shape
1221
+ self.second_last_token_hidden_states = torch.cat([
1222
+ self.second_last_token_hidden_states,
1223
+ torch.zeros_like(hidden_states)
1224
+ if second_last_token_hidden_states is None else
1225
+ second_last_token_hidden_states
1226
+ ])
1227
+
1228
def prune(self,
1229
seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
1205
- """Prune to provided list of sequence ids."""
1230
+ """Prune to provided list of sequence ids. Only used for decode steps.
1231
+ """
1232
+ # Currently this prunes all seq_ids not present in
1233
+ # seq_group_metadata_list which might cause problems where a sequence
1234
+ # may be "paused" then "resumed" later. This should only prune sequences
1235
+ # which are confirmed to be aborted.
1236
seq_ids = get_all_seq_ids(seq_group_metadata_list)
1237
if seq_ids != self._seq_ids:
1238
# Batch contents changed - prune removed sequences.
1239
index = [self._seq_ids.index(seq_id) for seq_id in seq_ids]
1240
self.hidden_states = self.hidden_states[index]
1241
+ if self.second_last_token_hidden_states is not None:
1242
+ self.second_last_token_hidden_states = self\
1243
+ .second_last_token_hidden_states[index]
1244
self._seq_ids = seq_ids
1245
1246
+ def expand_with_bonus_tokens(
1247
+ self, seq_with_bonus_token_in_last_step: set) -> None:
1248
+ """Expand hidden states for sequences with bonus tokens. This is in
1249
+ alignment with `MultiStepWorker._expand_execute_model_request`."""
1250
+ if self.second_last_token_hidden_states is None \
1251
+ or not seq_with_bonus_token_in_last_step:
1252
+ return
1253
+
1254
+ index = []
1255
+ for seq_id in self._seq_ids:
1256
+ i = self._seq_ids.index(seq_id)
1257
+ if seq_id in seq_with_bonus_token_in_last_step:
1258
+ index.append(i + len(self._seq_ids))
1259
+ index.append(i)
1260
+
1261
+ self.hidden_states = torch.cat(
1262
+ [self.hidden_states, self.second_last_token_hidden_states])[index]
1263
+
1264
1265
class ExecuteModelRequest(
1266
msgspec.Struct,
vllm/spec_decode/draft_model_runner.py
+19
@@ -203,6 +203,7 @@ class TP1DraftModelRunner(ModelRunner):
203
self,
204
model_input: ModelInputForGPUWithSamplingMetadata,
205
kv_caches: List[torch.Tensor],
206
+ previous_hidden_states: Optional[torch.Tensor] = None,
207
intermediate_tensors: Optional[IntermediateTensors] = None,
208
num_steps: int = 1,
209
) -> Optional[List[SamplerOutput]]:
@@ -280,13 +281,30 @@ class TP1DraftModelRunner(ModelRunner):
281
graph_batch_size = model_input.input_tokens.shape[0]
282
model_executable = (self.graph_runners[model_input.virtual_engine]
283
[graph_batch_size])
284
+
285
+ if previous_hidden_states is not None:
286
+ hidden_states = torch.cat([
287
+ previous_hidden_states,
288
+ torch.empty([
289
+ graph_batch_size - previous_hidden_states.shape[0],
290
+ *previous_hidden_states.shape[1:]
291
+ ],
292
+ dtype=previous_hidden_states.dtype,
293
+ device=previous_hidden_states.device)
294
+ ])
295
+ else:
296
+ hidden_states = None
297
else:
298
model_executable = self.model
299
+ hidden_states = previous_hidden_states
300
301
outputs: List[SamplerOutput] = []
302
for step in range(num_steps):
303
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
304
305
+ kwargs = {"previous_hidden_states": hidden_states} \
306
+ if previous_hidden_states is not None else {}
307
+
308
# Run model
309
hidden_states = model_executable(
310
input_ids=model_input.input_tokens,
@@ -296,6 +314,7 @@ class TP1DraftModelRunner(ModelRunner):
314
intermediate_tensors=intermediate_tensors,
315
**MultiModalInputs.as_kwargs(multi_modal_kwargs,
316
device=self.device),
317
+ **kwargs,
318
)
319
320
# Compute the logits.
vllm/spec_decode/multi_step_worker.py
+8
-2
@@ -4,8 +4,8 @@ from typing import Dict, List, Set, Tuple
4
5
import torch
6
7
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8
- SequenceGroupMetadata)
7
+from vllm.sequence import (ExecuteModelRequest, HiddenStates, SamplerOutput,
8
+ SequenceData, SequenceGroupMetadata)
9
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
10
from vllm.spec_decode.interfaces import (SpeculativeProposals,
11
SpeculativeProposer)
@@ -157,6 +157,12 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
157
158
updated_execute_model_req.seq_group_metadata_list =\
159
updated_seq_group_metadata_list
160
+
161
+ if isinstance(updated_execute_model_req.previous_hidden_states,
162
+ HiddenStates):
163
+ updated_execute_model_req.previous_hidden_states\
164
+ .expand_with_bonus_tokens(seq_with_bonus_token_in_last_step)
165
+
166
return updated_execute_model_req, indices_of_original_sequence_groups
167
168
@staticmethod
vllm/spec_decode/spec_decode_worker.py
+72
-25
@@ -147,6 +147,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
147
draft_worker_kwargs[
148
"model_runner_cls"] = TP1DraftModelRunner
149
else:
150
+ if draft_worker_kwargs[
151
+ "model_config"].hf_config.model_type == "eagle":
152
+ raise NotImplementedError(
153
+ "EAGLE does not support TP > 1 yet")
154
+
155
allow_zero_draft_token_step = False
156
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
157
@@ -355,14 +360,34 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
360
execute_model_req)
361
num_lookahead_slots = execute_model_req.num_lookahead_slots
362
363
+ # Speculative decoding is disabled in the following cases:
364
+ # 1. Prefill phase: Speculative decoding is not
365
+ # used during the prefill phase.
366
+ # 2. Auto-disable enabled: The running queue size exceeds
367
+ # the specified threshold.
368
+ # 3. No request: There are no requests in the batch.
369
+ # In any of these cases, the proposer and scorer workers
370
+ # are called normally.
371
+ no_spec = num_lookahead_slots == 0 or len(
372
+ execute_model_req.seq_group_metadata_list
373
+ ) == 0 or disable_all_speculation
374
+
375
# Broadcast how many lookahead slots are scheduled for this step, and
376
# whether all speculation is disabled, to all non-driver workers.
377
378
# This is required as if the number of draft model runs changes
379
# dynamically, the non-driver workers won't know unless we perform a
380
# communication to inform them.
381
+
382
+ # no_spec is used to signal non-driver worker about prefill vs decode
383
+ # stage. This is needed to ensure that order of execution of proposer
384
+ # and scorer is same in both driver and non-driver workers (i.e.,
385
+ # scorer -> proposer for prefill and proposer -> scorer in decode). This
386
+ # order is needed to support models like EAGLE that take scorer states
387
+ # as inputs.
388
broadcast_dict = dict(
389
num_lookahead_slots=num_lookahead_slots,
390
+ no_spec=no_spec,
391
disable_all_speculation=disable_all_speculation,
392
)
393
broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
@@ -373,17 +398,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
398
self._maybe_disable_speculative_tokens(
399
disable_all_speculation, execute_model_req.seq_group_metadata_list)
400
376
- # Speculative decoding is disabled in the following cases:
377
- # 1. Prefill phase: Speculative decoding is not
378
- # used during the prefill phase.
379
- # 2. Auto-disable enabled: The running queue size exceeds
380
- # the specified threshold.
381
- # 3. No request: There are no requests in the batch.
382
- # In any of these cases, the proposer and scorer workers
383
- # are called normally.
384
- if num_lookahead_slots == 0 or len(
385
- execute_model_req.seq_group_metadata_list
386
- ) == 0 or disable_all_speculation:
401
+ if no_spec:
402
return self._run_no_spec(execute_model_req,
403
skip_proposer=disable_all_speculation)
404
return self._run_speculative_decoding_step(execute_model_req,
@@ -464,8 +479,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
479
not called, meaning that the kv-cache in proposer for requests is not
480
updated, so they cannot enable spec decode in the rest decoding.
481
"""
467
- if not skip_proposer:
468
- self.proposer_worker.execute_model(execute_model_req)
482
483
sampler_output = self.scorer_worker.execute_model(execute_model_req)
484
assert len(sampler_output) == 1
@@ -476,10 +489,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
489
if hidden_states is not None:
490
if self.previous_hidden_states is None:
491
self.previous_hidden_states = HiddenStates(
479
- execute_model_req.seq_group_metadata_list, hidden_states)
492
+ hidden_states, execute_model_req.seq_group_metadata_list)
493
else:
494
self.previous_hidden_states.update(
482
- execute_model_req.seq_group_metadata_list, hidden_states)
495
+ hidden_states, execute_model_req.seq_group_metadata_list)
496
+
497
+ if not skip_proposer:
498
+ # We prepare the prefill hidden states here so that there no
499
+ # additional complexity in worker for spec_decode vs non_spec_decode
500
+ # flow and execute_model doesn't need additional modifications.
501
+ execute_model_req.previous_hidden_states = \
502
+ prepare_prefill_hidden_states(
503
+ sampler_output.prefill_hidden_states)
504
+
505
+ self.proposer_worker.execute_model(execute_model_req)
506
507
sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
508
execute_model_req=execute_model_req, sampler_output=sampler_output)
@@ -507,15 +530,23 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
530
return False
531
num_lookahead_slots = data["num_lookahead_slots"]
532
510
- # Even if num_lookahead_slots is zero, we want to run the proposer model
511
- # as it may have KV.
512
- #
513
- # We run the proposer once per lookahead slot. In the future we should
514
- # delegate how many times it runs to the proposer.
515
- for _ in range(max(num_lookahead_slots, 1)):
516
- self.proposer_worker.execute_model()
533
+ # In case of prefill, scorer_worker has to be run before proposer so
534
+ # that the hidden states can be propagated to proposer when needed.
535
+ if data["no_spec"]:
536
+ self.scorer_worker.execute_model()
537
+
538
+ if not data["disable_all_speculation"]:
539
+ # Even if num_lookahead_slots is zero, we want to run the
540
+ # proposer model as it may have KV.
541
+ #
542
+ # We run the proposer once per lookahead slot. In the future we
543
+ # should delegate how many times it runs to the proposer.
544
+ for _ in range(max(num_lookahead_slots, 1)):
545
+ self.proposer_worker.execute_model()
546
+
547
+ if not data["no_spec"]:
548
+ self.scorer_worker.execute_model()
549
518
- self.scorer_worker.execute_model()
550
return True
551
552
@nvtx_range("spec_decode_worker._run_speculative_decoding_step")
@@ -546,6 +577,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
577
raise RuntimeError("Cannot handle cases where distributed draft "
578
"workers generate no tokens")
579
580
+ execute_model_req.previous_hidden_states = None
581
+
582
with Timer() as scoring_timer:
583
proposal_scores = self.scorer.score_proposals(
584
execute_model_req,
@@ -651,10 +684,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
684
accepted_index = accepted_token_ids + 1 # Convert -1 to 0
685
accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
686
index = accepted_index[:, None, None].expand(-1, 1, hs_size)
687
+ second_last_token_hidden_states = hidden_states[:, -2] # b x d
688
hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
689
# Store hidden states from target model for subsequent decode step
656
- self.previous_hidden_states = HiddenStates(seq_group_metadata_list,
657
- hidden_states)
690
+ self.previous_hidden_states = HiddenStates(
691
+ hidden_states, seq_group_metadata_list,
692
+ second_last_token_hidden_states)
693
694
return accepted_token_ids, logprobs
695
@@ -951,3 +986,15 @@ def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
986
(proposer_cache_block_size_bytes + scorer_cache_block_size_bytes))
987
988
return new_num_gpu_blocks
989
+
990
+
991
+def prepare_prefill_hidden_states(
992
+ prefill_hidden_states: torch.Tensor) -> HiddenStates:
993
+ # For prefill step in proposer, we run the model for N-1 tokens
994
+ # because Nth token will be processed in the first decode step. For
995
+ # N-1 tokens, the input should be 0:N-1 hidden states which should
996
+ # be concatanated with 1:N token (since output of scorer has to be
997
+ # the input for proposer). Therefore, we shift the hidden states to
998
+ # align n-1th hidden state with nth token.
999
+ return HiddenStates(prefill_hidden_states.roll(
1000
+ shifts=1, dims=0)) if prefill_hidden_states is not None else None
vllm/transformers_utils/config.py
+6
-4
@@ -11,10 +11,11 @@ from transformers.models.auto.modeling_auto import (
11
from vllm.envs import VLLM_USE_MODELSCOPE
12
from vllm.logger import init_logger
13
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
14
- InternVLChatConfig, JAISConfig,
15
- MedusaConfig, MLPSpeculatorConfig,
16
- MPTConfig, NemotronConfig,
17
- RWConfig, UltravoxConfig)
14
+ EAGLEConfig, InternVLChatConfig,
15
+ JAISConfig, MedusaConfig,
16
+ MLPSpeculatorConfig, MPTConfig,
17
+ NemotronConfig, RWConfig,
18
+ UltravoxConfig)
19
20
if VLLM_USE_MODELSCOPE:
21
from modelscope import AutoConfig
@@ -32,6 +33,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
33
"jais": JAISConfig,
34
"mlp_speculator": MLPSpeculatorConfig,
35
"medusa": MedusaConfig,
36
+ "eagle": EAGLEConfig,
37
"internvl_chat": InternVLChatConfig,
38
"nemotron": NemotronConfig,
39
"ultravox": UltravoxConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,5 +1,6 @@
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
from vllm.transformers_utils.configs.dbrx import DbrxConfig
3
+from vllm.transformers_utils.configs.eagle import EAGLEConfig
4
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
5
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
6
# `FalconConfig` class from the official HuggingFace transformers library.
@@ -20,6 +21,7 @@ __all__ = [
21
"InternVLChatConfig",
22
"JAISConfig",
23
"MedusaConfig",
24
+ "EAGLEConfig",
25
"MLPSpeculatorConfig",
26
"NemotronConfig",
27
"UltravoxConfig",
vllm/transformers_utils/configs/eagle.py
new
+49
@@ -0,0 +1,49 @@
1
+import os
2
+from typing import Optional, Union
3
+
4
+from transformers import AutoConfig, PretrainedConfig
5
+
6
+
7
+class EAGLEConfig(PretrainedConfig):
8
+ model_type = "eagle"
9
+
10
+ def __init__(self,
11
+ model: Union[PretrainedConfig, dict, None] = None,
12
+ truncated_vocab_size: Optional[int] = None,
13
+ **kwargs):
14
+
15
+ model_config = None if model is None else (AutoConfig.for_model(
16
+ **model) if isinstance(model, dict) else model)
17
+
18
+ for k, v in kwargs.items():
19
+ if k != "architectures" and k != "model_type" and hasattr(
20
+ model_config, k):
21
+ setattr(model_config, k, v)
22
+
23
+ self.model = model_config
24
+
25
+ if self.model is None:
26
+ self.truncated_vocab_size = None
27
+ else:
28
+ self.truncated_vocab_size = self.model.vocab_size if \
29
+ truncated_vocab_size is None else truncated_vocab_size
30
+
31
+ if "architectures" not in kwargs:
32
+ kwargs["architectures"] = ["EAGLEModel"]
33
+
34
+ super().__init__(**kwargs)
35
+
36
+ if self.model is not None:
37
+ for k, v in self.model.to_dict().items():
38
+ if not hasattr(self, k):
39
+ setattr(self, k, v)
40
+
41
+ @classmethod
42
+ def from_pretrained(
43
+ cls,
44
+ pretrained_model_name_or_path: Union[str, os.PathLike],
45
+ **kwargs,
46
+ ) -> "EAGLEConfig":
47
+ config_dict, kwargs = cls.get_config_dict(
48
+ pretrained_model_name_or_path, **kwargs)
49
+ return cls.from_dict(config_dict, **kwargs)
vllm/worker/model_runner.py
+34
-10
@@ -1,5 +1,6 @@
1
import dataclasses
2
import gc
3
+import inspect
4
import itertools
5
import time
6
import warnings
@@ -1192,6 +1193,18 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1193
max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
1194
input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1195
input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1196
+
1197
+ # Prepare dummy previous_hidden_states only if needed by the model.
1198
+ # This is used by draft models such as EAGLE.
1199
+ previous_hidden_states = None
1200
+ if "previous_hidden_states" in inspect.signature(
1201
+ self.model.forward).parameters:
1202
+ previous_hidden_states = torch.empty(
1203
+ [max_batch_size,
1204
+ self.model_config.get_hidden_size()],
1205
+ dtype=self.model_config.dtype,
1206
+ device=self.device)
1207
+
1208
intermediate_inputs = None
1209
if not get_pp_group().is_first_rank:
1210
intermediate_inputs = self.model.make_empty_intermediate_tensors(
@@ -1264,6 +1277,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1277
"stream":
1278
graph_capture_context.stream
1279
}
1280
+ if previous_hidden_states is not None:
1281
+ capture_inputs[
1282
+ "previous_hidden_states"] = previous_hidden_states[:
1283
+ batch_size]
1284
+
1285
if self.has_seqlen_agnostic:
1286
# Only used by Mamba-based models CUDA graph atm (Jamba)
1287
capture_inputs.update({
@@ -1462,6 +1480,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1480
if model_input.is_prompt:
1481
hidden_states = hidden_or_intermediate_states.index_select(
1482
0, indices)
1483
+ output.prefill_hidden_states = hidden_or_intermediate_states
1484
elif decode_meta.use_cuda_graph:
1485
hidden_states = hidden_or_intermediate_states[:len(indices)]
1486
else:
@@ -1510,11 +1529,11 @@ class CUDAGraphRunner:
1529
# Note one iteration is not enough for torch.jit.script
1530
for _ in range(_NUM_WARMUP_ITERS):
1531
self.model(
1513
- input_ids,
1514
- positions,
1515
- kv_caches,
1516
- attn_metadata,
1517
- intermediate_inputs,
1532
+ input_ids=input_ids,
1533
+ positions=positions,
1534
+ kv_caches=kv_caches,
1535
+ attn_metadata=attn_metadata,
1536
+ intermediate_tensors=intermediate_inputs,
1537
**kwargs,
1538
)
1539
torch.cuda.synchronize()
@@ -1523,11 +1542,11 @@ class CUDAGraphRunner:
1542
self._graph = torch.cuda.CUDAGraph()
1543
with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
1544
output_hidden_or_intermediate_states = self.model(
1526
- input_ids,
1527
- positions,
1528
- kv_caches,
1529
- attn_metadata,
1530
- intermediate_inputs,
1545
+ input_ids=input_ids,
1546
+ positions=positions,
1547
+ kv_caches=kv_caches,
1548
+ attn_metadata=attn_metadata,
1549
+ intermediate_tensors=intermediate_inputs,
1550
**kwargs,
1551
)
1552
if hidden_or_intermediate_states is not None:
@@ -1588,6 +1607,11 @@ class CUDAGraphRunner:
1607
if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1608
self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1609
**kwargs)
1610
+
1611
+ if "previous_hidden_states" in self.input_buffers:
1612
+ self.input_buffers["previous_hidden_states"].copy_(
1613
+ kwargs["previous_hidden_states"], non_blocking=True)
1614
+
1615
if intermediate_tensors is not None:
1616
for key in intermediate_tensors.tensors:
1617
if key != "model_execute_time" and key != "model_forward_time":
vllm/worker/multi_step_worker.py
+13
-8
@@ -1,5 +1,7 @@
1
from dataclasses import dataclass
2
-from typing import List, Optional, Tuple
2
+from typing import Dict, List, Optional, Tuple
3
+
4
+import torch
5
6
from vllm.distributed import broadcast_tensor_dict, get_pp_group
7
from vllm.sequence import ExecuteModelRequest, SamplerOutput
@@ -43,7 +45,7 @@ class MultiStepWorker(Worker):
45
46
def _get_driver_input_and_broadcast(
47
self, execute_model_req: ExecuteModelRequest
46
- ) -> Tuple[BroadcastableModelInput, WorkerInput]:
48
+ ) -> Tuple[BroadcastableModelInput, WorkerInput, Dict[str, torch.Tensor]]:
49
"""
50
Get the driver input and broadcast it to other workers.
51
"""
@@ -85,7 +87,9 @@ class MultiStepWorker(Worker):
87
broadcast_data.update(model_input.as_broadcastable_tensor_dict())
88
broadcast_tensor_dict(broadcast_data, src=0)
89
88
- return model_input, worker_input
90
+ # Retuning empty dict here to keep this compatible with
91
+ # `LocalOrDistributedWorkerBase._get_driver_input_and_broadcast`
92
+ return model_input, worker_input, {}
93
94
def _prepare_last_sampled_token_ids_for_tp_workers(
95
self,
@@ -130,7 +134,8 @@ class MultiStepWorker(Worker):
134
def prepare_input(
135
self,
136
execute_model_req: Optional[ExecuteModelRequest] = None,
133
- ) -> Optional[Tuple[StatefulModelInput, WorkerInput]]:
137
+ ) -> Optional[Tuple[StatefulModelInput, WorkerInput, Dict[str,
138
+ torch.Tensor]]]:
139
"""
140
Depending on the current state of the request and multi step worker,
141
this method may skip the normal _prepare_model_input and
@@ -148,8 +153,8 @@ class MultiStepWorker(Worker):
153
return None
154
155
virtual_engine = execute_model_req.virtual_engine
151
- model_input, worker_input = self._get_driver_input_and_broadcast(
152
- execute_model_req)
156
+ (model_input, worker_input,
157
+ kwargs) = self._get_driver_input_and_broadcast(execute_model_req)
158
assert isinstance(model_input, StatefulModelInput)
159
if execute_model_req.is_first_multi_step:
160
# cache the worker input and model input for the next steps
@@ -162,7 +167,7 @@ class MultiStepWorker(Worker):
167
# loop
168
if broadcast_data is None:
169
return None
165
- model_input, worker_input = broadcast_data
170
+ model_input, worker_input, kwargs = broadcast_data
171
assert isinstance(model_input, StatefulModelInput)
172
virtual_engine = worker_input.virtual_engine
173
if model_input.is_first_multi_step:
@@ -186,4 +191,4 @@ class MultiStepWorker(Worker):
191
192
assert model_input is not None
193
assert worker_input is not None
189
- return model_input, worker_input
194
+ return model_input, worker_input, kwargs
vllm/worker/worker.py
+1
-1
@@ -86,7 +86,7 @@ class Worker(LocalOrDistributedWorkerBase):
86
or (speculative_config.draft_model_config.model ==
87
model_config.model) \
88
or (speculative_config.draft_model_config.hf_config.model_type
89
- not in ["medusa", "mlp_speculator"]) \
89
+ not in ["medusa", "mlp_speculator", "eagle"]) \
90
else {"return_hidden_states": True}
91
92
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
vllm/worker/worker_base.py
+50
-11
@@ -222,7 +222,9 @@ class LocalOrDistributedWorkerBase(WorkerBase):
222
raise NotImplementedError
223
224
def _get_worker_input_from_broadcast(
225
- self) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
225
+ self
226
+ ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput, Dict[
227
+ str, torch.Tensor]]]:
228
""" Get the worker input from the broadcasted tensor dict. """
229
assert self.do_metadata_broadcast
230
assert not self.is_driver_worker
@@ -235,11 +237,13 @@ class LocalOrDistributedWorkerBase(WorkerBase):
237
self.model_runner.make_model_input_from_broadcasted_tensor_dict(
238
broadcast_data))
239
238
- return model_input, worker_input
240
+ kwargs = extract_previous_hidden_states(broadcast_data)
241
+
242
+ return model_input, worker_input, kwargs
243
244
def _get_driver_input_and_broadcast(
245
self, execute_model_req: ExecuteModelRequest
242
- ) -> Tuple[BroadcastableModelInput, WorkerInput]:
246
+ ) -> Tuple[BroadcastableModelInput, WorkerInput, Dict[str, torch.Tensor]]:
247
""" Get the driver input and broadcast it to other workers. """
248
assert self.is_driver_worker
249
@@ -251,17 +255,21 @@ class LocalOrDistributedWorkerBase(WorkerBase):
255
execute_model_req.virtual_engine,
256
execute_model_req.finished_requests_ids))
257
258
+ kwargs = extract_previous_hidden_states(execute_model_req)
259
+
260
if self.do_metadata_broadcast:
261
broadcast_data = worker_input.as_broadcastable_tensor_dict()
262
broadcast_data.update(model_input.as_broadcastable_tensor_dict())
263
+ broadcast_data.update(kwargs)
264
broadcast_tensor_dict(broadcast_data, src=0)
265
259
- return model_input, worker_input
266
+ return model_input, worker_input, kwargs
267
268
def prepare_input(
269
self,
270
execute_model_req: Optional[ExecuteModelRequest] = None
264
- ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
271
+ ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput, Dict[
272
+ str, torch.Tensor]]]:
273
"""
274
Prepare the inputs to ModelRunner and workers.
275
"""
@@ -291,7 +299,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
299
if inputs is None:
300
return None
301
294
- model_input, worker_input = inputs
302
+ model_input, worker_input, kwargs = inputs
303
num_steps = worker_input.num_steps
304
305
self.execute_worker(worker_input)
@@ -312,9 +320,14 @@ class LocalOrDistributedWorkerBase(WorkerBase):
320
"model_execute_time", torch.tensor(0)).item()
321
322
output = self.model_runner.execute_model(
315
- model_input, self.kv_cache[worker_input.virtual_engine]
316
- if self.kv_cache is not None else None, intermediate_tensors,
317
- num_steps)
323
+ model_input=model_input,
324
+ kv_caches=self.kv_cache[worker_input.virtual_engine]
325
+ if self.kv_cache is not None else None,
326
+ intermediate_tensors=intermediate_tensors,
327
+ num_steps=num_steps,
328
+ **kwargs,
329
+ )
330
+
331
model_execute_time = time.perf_counter() - start_time
332
if not get_pp_group().is_last_rank:
333
# output is IntermediateTensors
@@ -360,9 +373,15 @@ class LocalOrDistributedWorkerBase(WorkerBase):
373
if worker_input.num_seq_groups == 0:
374
return []
375
376
+ kwargs = extract_previous_hidden_states(execute_model_req)
377
+
378
return self.model_runner.execute_model(
364
- model_input, self.kv_cache[worker_input.virtual_engine]
365
- if self.kv_cache is not None else None, intermediate_tensors)
379
+ model_input=model_input,
380
+ kv_caches=self.kv_cache[worker_input.virtual_engine]
381
+ if self.kv_cache is not None else None,
382
+ intermediate_tensors=intermediate_tensors,
383
+ **kwargs,
384
+ )
385
386
387
class WorkerWrapperBase:
@@ -439,3 +458,23 @@ class WorkerWrapperBase:
458
"This might cause deadlock in distributed execution.")
459
logger.exception(msg)
460
raise e
461
+
462
+
463
+def extract_previous_hidden_states(
464
+ data: Union[ExecuteModelRequest, Dict[str, torch.Tensor]]) -> \
465
+ Dict[str, torch.Tensor]:
466
+ """If data contains previous_hidden_states, extract it. This returns a dict
467
+ which can be used directly as additional kwargs in any following
468
+ execute_model calls. This is used in draft models like EAGLE."""
469
+ output = {}
470
+
471
+ # When called from non-driver worker, data is dict but when called from
472
+ # driver worker, data is ExecuteModelRequest.
473
+ if isinstance(data, dict):
474
+ if "previous_hidden_states" in data:
475
+ output["previous_hidden_states"] = data["previous_hidden_states"]
476
+ elif data.previous_hidden_states is not None:
477
+ output["previous_hidden_states"] = data.previous_hidden_states\
478
+ .hidden_states
479
+
480
+ return output