@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a3fce56b88
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Speculative Decoding] EAGLE Implementation with Top-1 proposer (#6830)

Abhinav Goyal committed Aug 22, 2024 at 15:12 UTC a3fce56b887cba61b8e05d50f01be4309ab2c42c
17 files changed +854 -83
tests/spec_decode/e2e/test_eagle_correctness.py new
+268
@@ -0,0 +1,268 @@
1 +"""This docstring details important information on the testing methodology.
2 +
3 +Most of the tests rely on "greedy equality", where we expect the output of
4 +speculative decoding on a sequence to exactly match the output of normal non-
5 +speculative decoding.
6 +
7 +Since speculative decoding with rejection sampling guarantees that the output
8 +distribution matches the target model's output distribution (up to hardware
9 +numerics, see https://arxiv.org/pdf/2302.01318.pdf), we can expect greedy
10 +equality.
11 +
12 +However, we still need to verify below scenario could be passed:
13 + * Batch size 1 greedy equality
14 + * Batch size >1 greedy equality
15 + * Test greedy equality under preemption
16 + * Test greedy equality under various number of speculative tokens.
17 +
18 +With those tests, we can say at least, EAGLE would not break the
19 +correctess for the target model outputs.
20 +"""
21 +
22 +import pytest
23 +
24 +from .conftest import run_greedy_equality_correctness_test
25 +
26 +# main model
27 +MAIN_MODEL = "JackFram/llama-68m"
28 +
29 +# speculative model
30 +SPEC_MODEL = "abhigoyal/vllm-eagle-llama-68m-random"
31 +
32 +# max. number of speculative tokens: this corresponds to
33 +# num_heads in the config.json of the speculator model.
34 +MAX_SPEC_TOKENS = 4
35 +
36 +# precision
37 +PRECISION = "float32"
38 +
39 +
40 +@pytest.mark.parametrize(
41 + "common_llm_kwargs",
42 + [{
43 + # Skip cuda graph recording for fast test.
44 + "enforce_eager": True,
45 +
46 + # Required for spec decode.
47 + "use_v2_block_manager": True,
48 +
49 + # Print spec metrics.
50 + "disable_log_stats": False,
51 +
52 + # Precision
53 + "dtype": PRECISION,
54 +
55 + # Main model
56 + "model": MAIN_MODEL,
57 + }])
58 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
59 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
60 +@pytest.mark.parametrize("test_llm_kwargs", [
61 + {
62 + "speculative_model": SPEC_MODEL,
63 + "num_speculative_tokens": MAX_SPEC_TOKENS,
64 + },
65 +])
66 +@pytest.mark.parametrize("output_len", [
67 + 128,
68 +])
69 +@pytest.mark.parametrize("batch_size", [1, 32])
70 +@pytest.mark.parametrize("seed", [1])
71 +def test_eagle_e2e_greedy_correctness(baseline_llm_generator,
72 + test_llm_generator, batch_size: int,
73 + output_len: int):
74 + """Verify greedy equality with different batch size."""
75 + run_greedy_equality_correctness_test(baseline_llm_generator,
76 + test_llm_generator,
77 + batch_size,
78 + max_output_len=output_len,
79 + force_output_len=True)
80 +
81 +
82 +@pytest.mark.parametrize(
83 + "common_llm_kwargs",
84 + [{
85 + "enforce_eager": False,
86 +
87 + # Required for spec decode.
88 + "use_v2_block_manager": True,
89 +
90 + # Print spec metrics.
91 + "disable_log_stats": False,
92 +
93 + # Precision
94 + "dtype": PRECISION,
95 +
96 + # Main model
97 + "model": MAIN_MODEL,
98 + }])
99 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
100 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
101 +@pytest.mark.parametrize("test_llm_kwargs", [
102 + {
103 + "speculative_model": SPEC_MODEL,
104 + "num_speculative_tokens": MAX_SPEC_TOKENS,
105 + },
106 +])
107 +@pytest.mark.parametrize("output_len", [
108 + 128,
109 +])
110 +@pytest.mark.parametrize("batch_size", [1, 32])
111 +@pytest.mark.parametrize("seed", [1])
112 +def test_eagle_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
113 + test_llm_generator,
114 + batch_size: int,
115 + output_len: int):
116 + """Verify greedy equality with cuda graph enabled and different
117 + batch sizes."""
118 + run_greedy_equality_correctness_test(baseline_llm_generator,
119 + test_llm_generator,
120 + batch_size,
121 + max_output_len=output_len,
122 + force_output_len=True)
123 +
124 +
125 +@pytest.mark.parametrize(
126 + "common_llm_kwargs",
127 + [{
128 + "block_size": 8,
129 + # 2 for small prompt, 256//8 for generated.
130 + "num_gpu_blocks_override": 2 + 256 // 8,
131 + "max_model_len": (2 + 256 // 8) * 8,
132 +
133 + # Skip cuda graph recording for fast test.
134 + "enforce_eager": True,
135 +
136 + # Required for spec decode.
137 + "use_v2_block_manager": True,
138 +
139 + # Precision
140 + "dtype": PRECISION,
141 +
142 + # Main model
143 + "model": MAIN_MODEL,
144 + }])
145 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
146 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
147 +@pytest.mark.parametrize("test_llm_kwargs", [
148 + {
149 + "speculative_model": SPEC_MODEL,
150 + "num_speculative_tokens": MAX_SPEC_TOKENS,
151 + },
152 +])
153 +@pytest.mark.parametrize(
154 + "output_len",
155 + [
156 + # Use small output len for fast test.
157 + 128,
158 + ])
159 +@pytest.mark.parametrize("batch_size", [4])
160 +@pytest.mark.parametrize("seed", [1])
161 +def test_eagle_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
162 + test_llm_generator,
163 + batch_size: int,
164 + output_len: int):
165 + """Verify greedy equality, even when some sequences are preempted mid-
166 + generation.
167 + """
168 + run_greedy_equality_correctness_test(baseline_llm_generator,
169 + test_llm_generator,
170 + batch_size,
171 + max_output_len=output_len,
172 + force_output_len=True)
173 +
174 +
175 +@pytest.mark.parametrize(
176 + "common_llm_kwargs",
177 + [{
178 + # Skip cuda graph recording for fast test.
179 + "enforce_eager": True,
180 +
181 + # Required for spec decode.
182 + "use_v2_block_manager": True,
183 +
184 + # Precision
185 + "dtype": PRECISION,
186 +
187 + # Main model
188 + "model": MAIN_MODEL,
189 + }])
190 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
191 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
192 +@pytest.mark.parametrize(
193 + "test_llm_kwargs",
194 + [
195 + {
196 + "speculative_model": SPEC_MODEL,
197 + "num_speculative_tokens": k,
198 + }
199 + # Try a range of num. speculative tokens
200 + for k in range(1, 1 + MAX_SPEC_TOKENS)
201 + ])
202 +@pytest.mark.parametrize("batch_size", [2])
203 +@pytest.mark.parametrize(
204 + "output_len",
205 + [
206 + # Use smaller output len for fast test.
207 + 32,
208 + ])
209 +@pytest.mark.parametrize("seed", [1])
210 +def test_eagle_different_k(baseline_llm_generator, test_llm_generator,
211 + batch_size: int, output_len: int):
212 + """Verify that eagle speculative decoding produces exact equality
213 + to without spec decode with different values of num_speculative_tokens.
214 + """
215 + run_greedy_equality_correctness_test(baseline_llm_generator,
216 + test_llm_generator,
217 + batch_size,
218 + max_output_len=output_len,
219 + force_output_len=True)
220 +
221 +
222 +@pytest.mark.parametrize(
223 + "common_llm_kwargs",
224 + [{
225 + # Skip cuda graph recording for fast test.
226 + "enforce_eager": True,
227 +
228 + # Required for spec decode.
229 + "use_v2_block_manager": True,
230 +
231 + # Precision
232 + "dtype": PRECISION,
233 +
234 + # Main model
235 + "model": MAIN_MODEL,
236 + }])
237 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
238 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
239 +@pytest.mark.parametrize("test_llm_kwargs",
240 + [{
241 + "speculative_model": SPEC_MODEL,
242 + "num_speculative_tokens": MAX_SPEC_TOKENS,
243 + "speculative_disable_by_batch_size": 4
244 + }])
245 +@pytest.mark.parametrize("batch_size", [1, 5])
246 +@pytest.mark.parametrize(
247 + "output_len",
248 + [
249 + # Use smaller output len for fast test.
250 + 32,
251 + ])
252 +@pytest.mark.parametrize("seed", [1])
253 +def test_eagle_disable_queue(baseline_llm_generator, test_llm_generator,
254 + batch_size: int, output_len: int):
255 + """Verify that eagle speculative decoding produces exact equality
256 + to without spec decode when speculation is disabled for large
257 + batch sizes.
258 + """
259 + run_greedy_equality_correctness_test(baseline_llm_generator,
260 + test_llm_generator,
261 + batch_size,
262 + max_output_len=output_len,
263 + force_output_len=True)
264 +
265 +
266 +if __name__ == "__main__":
267 + import pytest
268 + pytest.main([__file__])
tests/spec_decode/e2e/test_medusa_correctness.py
+56 -12
@@ -70,8 +70,9 @@ PRECISION = "float32"
70 ])
71 @pytest.mark.parametrize("batch_size", [1, 32])
72 @pytest.mark.parametrize("seed", [1])
73 -def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
74 - batch_size: int, output_len: int):
73 +def test_medusa_e2e_greedy_correctness(baseline_llm_generator,
74 + test_llm_generator, batch_size: int,
75 + output_len: int):
76 """Verify greedy equality with different batch size."""
77 run_greedy_equality_correctness_test(baseline_llm_generator,
78 test_llm_generator,
@@ -80,6 +81,49 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
81 force_output_len=True)
82
83
84 +@pytest.mark.parametrize(
85 + "common_llm_kwargs",
86 + [{
87 + "enforce_eager": False,
88 +
89 + # Required for spec decode.
90 + "use_v2_block_manager": True,
91 +
92 + # Print spec metrics.
93 + "disable_log_stats": False,
94 +
95 + # Precision
96 + "dtype": PRECISION,
97 +
98 + # Main model
99 + "model": MAIN_MODEL,
100 + }])
101 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
102 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
103 +@pytest.mark.parametrize("test_llm_kwargs", [
104 + {
105 + "speculative_model": SPEC_MODEL,
106 + "num_speculative_tokens": MAX_SPEC_TOKENS,
107 + },
108 +])
109 +@pytest.mark.parametrize("output_len", [
110 + 128,
111 +])
112 +@pytest.mark.parametrize("batch_size", [1, 32])
113 +@pytest.mark.parametrize("seed", [1])
114 +def test_medusa_e2e_greedy_correctness_cuda_graph(baseline_llm_generator,
115 + test_llm_generator,
116 + batch_size: int,
117 + output_len: int):
118 + """Verify greedy equality with cuda graph enabled and different
119 + batch sizes."""
120 + run_greedy_equality_correctness_test(baseline_llm_generator,
121 + test_llm_generator,
122 + batch_size,
123 + max_output_len=output_len,
124 + force_output_len=True)
125 +
126 +
127 @pytest.mark.parametrize(
128 "common_llm_kwargs",
129 [{
@@ -116,10 +160,10 @@ def test_mlp_e2e_greedy_correctness(baseline_llm_generator, test_llm_generator,
160 ])
161 @pytest.mark.parametrize("batch_size", [4])
162 @pytest.mark.parametrize("seed", [1])
119 -def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
120 - test_llm_generator,
121 - batch_size: int,
122 - output_len: int):
163 +def test_medusa_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
164 + test_llm_generator,
165 + batch_size: int,
166 + output_len: int):
167 """Verify greedy equality, even when some sequences are preempted mid-
168 generation.
169 """
@@ -165,9 +209,9 @@ def test_mlp_e2e_greedy_correctness_with_preemption(baseline_llm_generator,
209 32,
210 ])
211 @pytest.mark.parametrize("seed", [1])
168 -def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
169 - batch_size: int, output_len: int):
170 - """Verify that mlp speculative decoding produces exact equality
212 +def test_medusa_different_k(baseline_llm_generator, test_llm_generator,
213 + batch_size: int, output_len: int):
214 + """Verify that medusa speculative decoding produces exact equality
215 to without spec decode with different values of num_speculative_tokens.
216 """
217 run_greedy_equality_correctness_test(baseline_llm_generator,
@@ -208,9 +252,9 @@ def test_mlp_different_k(baseline_llm_generator, test_llm_generator,
252 32,
253 ])
254 @pytest.mark.parametrize("seed", [1])
211 -def test_mlp_disable_queue(baseline_llm_generator, test_llm_generator,
212 - batch_size: int, output_len: int):
213 - """Verify that mlp speculative decoding produces exact equality
255 +def test_medusa_disable_queue(baseline_llm_generator, test_llm_generator,
256 + batch_size: int, output_len: int):
257 + """Verify that medusa speculative decoding produces exact equality
258 to without spec decode when speculation is disabled for large
259 batch sizes.
260 """
tests/spec_decode/test_multi_step_worker.py
+35 -1
@@ -6,7 +6,8 @@ import pytest
6 import torch
7
8 from vllm.model_executor.utils import set_random_seed
9 -from vllm.sequence import ExecuteModelRequest, Logprob, SamplerOutput
9 +from vllm.sequence import (ExecuteModelRequest, HiddenStates, Logprob,
10 + SamplerOutput, get_all_seq_ids)
11 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
12 from vllm.spec_decode.multi_step_worker import MultiStepWorker
13 from vllm.spec_decode.top1_proposer import Top1Proposer
@@ -690,3 +691,36 @@ def test_use_draft_model_runner_advance_step():
691 worker.execute_model(execute_model_req=execute_model_req)
692 call_args_list = worker.model_runner._gpu_advance_step.call_args_list
693 assert len(call_args_list) == 1
694 +
695 +
696 +@torch.inference_mode()
697 +def test_expand_execute_model_request_sync_with_expand_hidden_states():
698 + """
699 + In this test we verify that the logic for expanding the
700 + seq_group_metadata_list remains in sync with the expansion logic of
701 + the HiddenStates in _expand_execute_model_request.
702 + """
703 + k = 5
704 + batch_size = 16
705 + seq_with_bonus_token_in_last_step = [1, 3, 8, 10, 13, 15]
706 +
707 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
708 +
709 + execute_model_request = ExecuteModelRequest(
710 + seq_group_metadata_list,
711 + previous_hidden_states=HiddenStates(
712 + torch.arange(batch_size), seq_group_metadata_list,
713 + torch.arange(batch_size, 2 * batch_size)))
714 +
715 + expanded_execute_model_request, orig_seq_group_ids = MultiStepWorker.\
716 + _expand_execute_model_request(execute_model_request,
717 + seq_with_bonus_token_in_last_step)
718 +
719 + all_seq_ids = torch.tensor(
720 + get_all_seq_ids(
721 + expanded_execute_model_request.seq_group_metadata_list))
722 + ref_expanded_hidden_states = all_seq_ids + batch_size
723 + ref_expanded_hidden_states[orig_seq_group_ids] -= batch_size
724 +
725 + assert (ref_expanded_hidden_states == expanded_execute_model_request.
726 + previous_hidden_states.hidden_states).all().item()
vllm/model_executor/models/__init__.py
+1
@@ -60,6 +60,7 @@ _GENERATION_MODELS = {
60 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
61 "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
62 "MedusaModel": ("medusa", "Medusa"),
63 + "EAGLEModel": ("eagle", "EAGLE"),
64 "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
65 "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
66 }
vllm/model_executor/models/eagle.py new
+161
@@ -0,0 +1,161 @@
1 +from typing import Iterable, List, Optional, Tuple
2 +
3 +import torch
4 +import torch.nn as nn
5 +
6 +from vllm.attention.backends.abstract import AttentionMetadata
7 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
8 +from vllm.model_executor.layers.vocab_parallel_embedding import (
9 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
10 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
11 +from vllm.model_executor.models import ModelRegistry
12 +from vllm.model_executor.sampling_metadata import SamplingMetadata
13 +from vllm.sequence import IntermediateTensors, SamplerOutput
14 +from vllm.transformers_utils.configs.eagle import EAGLEConfig
15 +
16 +
17 +class EAGLE(nn.Module):
18 + """This class implements the EAGLE draft model from the paper: https://arxiv.org/pdf/2401.15077
19 + Reference implementation: https://github.com/SafeAILab/EAGLE
20 +
21 + Differences from reference implementation:
22 + 1. In reference, LlamaDecoderLayer implementation doesn't have
23 + input_layernorm for 1st decoder layer (https://github.com/SafeAILab/EAGLE/blob/7d065d084443fbfd386f88839efd7193c12be869/eagle/model/cnets.py#L427)
24 + but we do as HF implementation also does.
25 + 2. We allow any decoder layer to be used in EAGLE whereas in reference
26 + decoder layer is fixed to be LlamaDecoderLayer.
27 + 3. We have an optional token_map which reduces draft vocab to most
28 + frequently used tokens to give some additional speed-up by reducing
29 + sampling overhead. This is disabled unless the checkpoint file has
30 + explicit token_map tensor and config has an optional attribute
31 + truncated_vocab_size < vocab_size. To use this technique, one has to find
32 + the top-k most frequent tokens in target dataset and add that as a tensor
33 + in the draft checkpoint (using key token_map). Also, the draft config
34 + needs to have truncated_vocab_size (=k) as an attribute."""
35 +
36 + def __init__(self, config: EAGLEConfig, *args, **kwargs) -> None:
37 + super().__init__()
38 + self.config = config
39 +
40 + architectures = getattr(self.config.model, "architectures", [])
41 + model_cls, _ = ModelRegistry.resolve_model_cls(architectures)
42 +
43 + self.model = model_cls(self.config.model, *args, **kwargs)
44 + self.fc = nn.Linear(config.model.hidden_size * 2,
45 + config.model.hidden_size,
46 + bias=False)
47 +
48 + self.orig_vocab_size = config.vocab_size
49 + self.truncated_vocab_size = config.truncated_vocab_size
50 + self.unpadded_vocab_size = self.truncated_vocab_size
51 +
52 + self.lm_head = ParallelLMHead(
53 + self.unpadded_vocab_size,
54 + config.hidden_size,
55 + org_num_embeddings=self.truncated_vocab_size,
56 + padding_size=DEFAULT_VOCAB_PADDING_SIZE,
57 + )
58 +
59 + logit_scale = getattr(config, "logit_scale", 1.0)
60 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
61 + self.truncated_vocab_size,
62 + logit_scale)
63 +
64 + # Token map is a idx to token mapping to reduce the vocab size for
65 + # the draft model. Using smaller vocab size for draft, containing
66 + # only most frequent tokens reduces the speculation overhead. This
67 + # doesn't affect the acceptance rate much and thus gives more speed
68 + # -up. By default, this is disabled and is only used if the EAGLE
69 + # checkpoint file has token_map tensor.
70 + self.token_map = None
71 +
72 + @property
73 + def sampler(self):
74 + return self.model.sampler
75 +
76 + def forward(
77 + self,
78 + input_ids: torch.Tensor,
79 + positions: torch.Tensor,
80 + kv_caches: List[torch.Tensor],
81 + attn_metadata: AttentionMetadata,
82 + previous_hidden_states: torch.Tensor,
83 + intermediate_tensors: Optional[IntermediateTensors] = None,
84 + ) -> torch.Tensor:
85 +
86 + tok_embeds = self.model.model.embed_tokens(input_ids)
87 + inputs_embeds = self.fc(
88 + torch.cat([tok_embeds, previous_hidden_states], dim=-1))
89 +
90 + inputs_embeds[positions == 0] = 0 # masking inputs at position=0
91 +
92 + hidden_states = self.model.model(
93 + input_ids=None,
94 + inputs_embeds=inputs_embeds,
95 + positions=positions,
96 + kv_caches=kv_caches,
97 + attn_metadata=attn_metadata,
98 + intermediate_tensors=intermediate_tensors)
99 + return hidden_states
100 +
101 + def compute_logits(self, hidden_states: torch.Tensor,
102 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
103 + logits = self.logits_processor(self.lm_head, hidden_states,
104 + sampling_metadata)
105 +
106 + if self.token_map is not None:
107 + _logits = logits
108 + logits = -torch.inf * torch.ones(
109 + size=(*_logits.shape[:-1], self.orig_vocab_size),
110 + device=_logits.device,
111 + dtype=_logits.dtype)
112 +
113 + logits[..., self.token_map] = _logits
114 +
115 + return logits
116 +
117 + def sample(
118 + self,
119 + logits: torch.Tensor,
120 + sampling_metadata: SamplingMetadata,
121 + ) -> Optional[SamplerOutput]:
122 + next_tokens = self.sampler(logits, sampling_metadata)
123 + return next_tokens
124 +
125 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
126 + # This implementation is incompitable with https://huggingface.co/yuhuili/EAGLE-LLaMA3-Instruct-8B
127 + # due to missing lm_head weights and its config being that of a
128 + # Llama model. Here's a compatible version with the same weights:
129 + # https://huggingface.co/abhigoyal/EAGLE-LLaMA3-Instruct-8B-vllm
130 + # Also, here's an example script for converting trained EAGLE
131 + # checkpoint to vLLM compatible version: https://gist.github.com/abhigoyal1997/1e7a4109ccb7704fbc67f625e86b2d6d
132 + model_weights = {}
133 + for name, loaded_weight in weights:
134 + if name == "token_map":
135 + if self.config.truncated_vocab_size < self.config.vocab_size:
136 + self.token_map = nn.Parameter(loaded_weight,
137 + requires_grad=False)
138 + elif name.startswith("fc."):
139 + weight_loader = getattr(self.fc.weight, "weight_loader",
140 + default_weight_loader)
141 + weight_loader(self.fc.weight, loaded_weight)
142 + elif name.startswith("model.lm_head.") or name.startswith(
143 + "model.model."):
144 + model_weights[name.split("model.", 1)[-1]] = loaded_weight
145 + elif name.startswith("lm_head.") or name.startswith("model."):
146 + model_weights[name] = loaded_weight
147 + else:
148 + model_weights[f"model.{name}"] = loaded_weight
149 +
150 + lm_head_weight = model_weights.pop("lm_head.weight")
151 +
152 + if self.token_map is not None and\
153 + lm_head_weight.shape[0] > self.token_map.shape[0]:
154 +
155 + lm_head_weight = lm_head_weight[self.token_map]
156 +
157 + weight_loader = getattr(self.lm_head.weight, "weight_loader",
158 + default_weight_loader)
159 + weight_loader(self.lm_head.weight, lm_head_weight)
160 +
161 + self.model.load_weights(model_weights.items())
vllm/model_executor/models/medusa.py
+19
@@ -30,6 +30,19 @@ class ResidualBlock(nn.Module):
30
31
32 class Medusa(nn.Module):
33 + """This class implements the Medusa draft model from the paper: https://arxiv.org/abs/2401.10774
34 + Reference implementation: https://github.com/FasterDecoding/Medusa
35 +
36 + Differences from reference implementation:
37 + 1. Currently this only supports generating proposals from top-1 tokens.
38 + 2. We have an optional token_map which reduces draft vocab to most
39 + frequently used tokens to give some additional speed-up by reducing
40 + sampling overhead. This is disabled unless the checkpoint file has
41 + explicit token_map tensor and config has an optional attribute
42 + truncated_vocab_size < vocab_size. To use this technique, one has to find
43 + the top-k most frequent tokens in target dataset and add that as a tensor
44 + in the draft checkpoint (using key token_map). Also, the draft config
45 + needs to have truncated_vocab_size (=k) as an attribute."""
46
47 def __init__(self, config: MedusaConfig, **_) -> None:
48 super().__init__()
@@ -57,6 +70,12 @@ class Medusa(nn.Module):
70 self.truncated_vocab_size,
71 logit_scale)
72
73 + # Token map is a idx to token mapping to reduce the vocab size for
74 + # the draft model. Using smaller vocab size for draft, containing
75 + # only most frequent tokens reduces the speculation overhead. This
76 + # doesn't affect the acceptance rate much and thus gives more speed
77 + # -up. By default, this is disabled and is only used if the EAGLE
78 + # checkpoint file has token_map tensor.
79 self.token_map = None
80
81 def forward(self, hidden_states: torch.Tensor) -> List[torch.Tensor]:
vllm/sequence.py
+60 -9
@@ -1092,6 +1092,10 @@ class SamplerOutput(
1092 # Optional last hidden states from the model.
1093 hidden_states: Optional[torch.Tensor] = None
1094
1095 + # Optional prefill hidden states from the model
1096 + # (used for models like EAGLE).
1097 + prefill_hidden_states: Optional[torch.Tensor] = None
1098 +
1099 # Time taken in the forward pass for this across all workers
1100 model_forward_time: Optional[float] = None
1101
@@ -1176,40 +1180,87 @@ class HiddenStates(msgspec.Struct, array_like=True,
1180 omit_defaults=True): # type: ignore[call-arg]
1181 """Hidden states corresponding to in-progress sequences.
1182 Used in speculative decoding to pass hidden states from
1179 - the target model to the proposer model in the subsequent step.
1183 + the target model to the proposer model.
1184
1185 seq_ids are the sequence ids of each entry of the batch
1186 dimension of the hidden_states tensor"""
1183 -
1184 - seq_group_metadata_list: List[SequenceGroupMetadata]
1187 + # Scorer hidden states. For prefill step, it is used for hidden states of
1188 + # all tokens, whereas for decode step, it use used for last accepted tokens.
1189 hidden_states: torch.Tensor
1190 + # The sequence group metadata list. Only needed for decode step.
1191 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
1192 + # Scorer hidden states of the 2nd last token proposed by the proposer (
1193 + # irrespective of whether it was accepted or not). Only used for cases when
1194 + # last proposed token is accepted (i.e., in case of bonus tokens). For the
1195 + # case of no bonus tokens, these are ignored.
1196 + second_last_token_hidden_states: Optional[torch.Tensor] = None
1197 +
1198 _seq_ids: List[int] = msgspec.field(default_factory=list)
1199
1200 def __post_init__(self):
1189 - self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list)
1190 - assert len(self.seq_group_metadata_list) == len(self.hidden_states)
1201 + if self.seq_group_metadata_list is not None:
1202 + assert len(self.seq_group_metadata_list) == len(self.hidden_states)
1203 + self._seq_ids = get_all_seq_ids(self.seq_group_metadata_list)
1204
1205 @property
1206 def seq_ids(self) -> List[int]:
1207 return self._seq_ids
1208
1196 - def update(self, seq_group_metadata_list: List[SequenceGroupMetadata],
1197 - hidden_states: torch.Tensor) -> None:
1198 - """Update hidden states from target model invocation."""
1209 + def update(self,
1210 + hidden_states: torch.Tensor,
1211 + seq_group_metadata_list: List[SequenceGroupMetadata],
1212 + second_last_token_hidden_states: Optional[torch.Tensor] = None):
1213 + """Update hidden states from target model invocation. Only used for
1214 + decode steps"""
1215 assert len(seq_group_metadata_list) == len(hidden_states)
1216 self._seq_ids.extend(get_all_seq_ids(seq_group_metadata_list))
1217 self.hidden_states = torch.cat([self.hidden_states, hidden_states])
1218
1219 + if self.second_last_token_hidden_states is not None:
1220 + # Adding dummy hidden_states to this to maintain same shape
1221 + self.second_last_token_hidden_states = torch.cat([
1222 + self.second_last_token_hidden_states,
1223 + torch.zeros_like(hidden_states)
1224 + if second_last_token_hidden_states is None else
1225 + second_last_token_hidden_states
1226 + ])
1227 +
1228 def prune(self,
1229 seq_group_metadata_list: List[SequenceGroupMetadata]) -> None:
1205 - """Prune to provided list of sequence ids."""
1230 + """Prune to provided list of sequence ids. Only used for decode steps.
1231 + """
1232 + # Currently this prunes all seq_ids not present in
1233 + # seq_group_metadata_list which might cause problems where a sequence
1234 + # may be "paused" then "resumed" later. This should only prune sequences
1235 + # which are confirmed to be aborted.
1236 seq_ids = get_all_seq_ids(seq_group_metadata_list)
1237 if seq_ids != self._seq_ids:
1238 # Batch contents changed - prune removed sequences.
1239 index = [self._seq_ids.index(seq_id) for seq_id in seq_ids]
1240 self.hidden_states = self.hidden_states[index]
1241 + if self.second_last_token_hidden_states is not None:
1242 + self.second_last_token_hidden_states = self\
1243 + .second_last_token_hidden_states[index]
1244 self._seq_ids = seq_ids
1245
1246 + def expand_with_bonus_tokens(
1247 + self, seq_with_bonus_token_in_last_step: set) -> None:
1248 + """Expand hidden states for sequences with bonus tokens. This is in
1249 + alignment with `MultiStepWorker._expand_execute_model_request`."""
1250 + if self.second_last_token_hidden_states is None \
1251 + or not seq_with_bonus_token_in_last_step:
1252 + return
1253 +
1254 + index = []
1255 + for seq_id in self._seq_ids:
1256 + i = self._seq_ids.index(seq_id)
1257 + if seq_id in seq_with_bonus_token_in_last_step:
1258 + index.append(i + len(self._seq_ids))
1259 + index.append(i)
1260 +
1261 + self.hidden_states = torch.cat(
1262 + [self.hidden_states, self.second_last_token_hidden_states])[index]
1263 +
1264
1265 class ExecuteModelRequest(
1266 msgspec.Struct,
vllm/spec_decode/draft_model_runner.py
+19
@@ -203,6 +203,7 @@ class TP1DraftModelRunner(ModelRunner):
203 self,
204 model_input: ModelInputForGPUWithSamplingMetadata,
205 kv_caches: List[torch.Tensor],
206 + previous_hidden_states: Optional[torch.Tensor] = None,
207 intermediate_tensors: Optional[IntermediateTensors] = None,
208 num_steps: int = 1,
209 ) -> Optional[List[SamplerOutput]]:
@@ -280,13 +281,30 @@ class TP1DraftModelRunner(ModelRunner):
281 graph_batch_size = model_input.input_tokens.shape[0]
282 model_executable = (self.graph_runners[model_input.virtual_engine]
283 [graph_batch_size])
284 +
285 + if previous_hidden_states is not None:
286 + hidden_states = torch.cat([
287 + previous_hidden_states,
288 + torch.empty([
289 + graph_batch_size - previous_hidden_states.shape[0],
290 + *previous_hidden_states.shape[1:]
291 + ],
292 + dtype=previous_hidden_states.dtype,
293 + device=previous_hidden_states.device)
294 + ])
295 + else:
296 + hidden_states = None
297 else:
298 model_executable = self.model
299 + hidden_states = previous_hidden_states
300
301 outputs: List[SamplerOutput] = []
302 for step in range(num_steps):
303 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
304
305 + kwargs = {"previous_hidden_states": hidden_states} \
306 + if previous_hidden_states is not None else {}
307 +
308 # Run model
309 hidden_states = model_executable(
310 input_ids=model_input.input_tokens,
@@ -296,6 +314,7 @@ class TP1DraftModelRunner(ModelRunner):
314 intermediate_tensors=intermediate_tensors,
315 **MultiModalInputs.as_kwargs(multi_modal_kwargs,
316 device=self.device),
317 + **kwargs,
318 )
319
320 # Compute the logits.
vllm/spec_decode/multi_step_worker.py
+8 -2
@@ -4,8 +4,8 @@ from typing import Dict, List, Set, Tuple
4
5 import torch
6
7 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8 - SequenceGroupMetadata)
7 +from vllm.sequence import (ExecuteModelRequest, HiddenStates, SamplerOutput,
8 + SequenceData, SequenceGroupMetadata)
9 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
10 from vllm.spec_decode.interfaces import (SpeculativeProposals,
11 SpeculativeProposer)
@@ -157,6 +157,12 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
157
158 updated_execute_model_req.seq_group_metadata_list =\
159 updated_seq_group_metadata_list
160 +
161 + if isinstance(updated_execute_model_req.previous_hidden_states,
162 + HiddenStates):
163 + updated_execute_model_req.previous_hidden_states\
164 + .expand_with_bonus_tokens(seq_with_bonus_token_in_last_step)
165 +
166 return updated_execute_model_req, indices_of_original_sequence_groups
167
168 @staticmethod
vllm/spec_decode/spec_decode_worker.py
+72 -25
@@ -147,6 +147,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
147 draft_worker_kwargs[
148 "model_runner_cls"] = TP1DraftModelRunner
149 else:
150 + if draft_worker_kwargs[
151 + "model_config"].hf_config.model_type == "eagle":
152 + raise NotImplementedError(
153 + "EAGLE does not support TP > 1 yet")
154 +
155 allow_zero_draft_token_step = False
156 proposer_worker = MultiStepWorker(**draft_worker_kwargs)
157
@@ -355,14 +360,34 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
360 execute_model_req)
361 num_lookahead_slots = execute_model_req.num_lookahead_slots
362
363 + # Speculative decoding is disabled in the following cases:
364 + # 1. Prefill phase: Speculative decoding is not
365 + # used during the prefill phase.
366 + # 2. Auto-disable enabled: The running queue size exceeds
367 + # the specified threshold.
368 + # 3. No request: There are no requests in the batch.
369 + # In any of these cases, the proposer and scorer workers
370 + # are called normally.
371 + no_spec = num_lookahead_slots == 0 or len(
372 + execute_model_req.seq_group_metadata_list
373 + ) == 0 or disable_all_speculation
374 +
375 # Broadcast how many lookahead slots are scheduled for this step, and
376 # whether all speculation is disabled, to all non-driver workers.
377
378 # This is required as if the number of draft model runs changes
379 # dynamically, the non-driver workers won't know unless we perform a
380 # communication to inform them.
381 +
382 + # no_spec is used to signal non-driver worker about prefill vs decode
383 + # stage. This is needed to ensure that order of execution of proposer
384 + # and scorer is same in both driver and non-driver workers (i.e.,
385 + # scorer -> proposer for prefill and proposer -> scorer in decode). This
386 + # order is needed to support models like EAGLE that take scorer states
387 + # as inputs.
388 broadcast_dict = dict(
389 num_lookahead_slots=num_lookahead_slots,
390 + no_spec=no_spec,
391 disable_all_speculation=disable_all_speculation,
392 )
393 broadcast_tensor_dict(broadcast_dict, src=self._driver_rank)
@@ -373,17 +398,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
398 self._maybe_disable_speculative_tokens(
399 disable_all_speculation, execute_model_req.seq_group_metadata_list)
400
376 - # Speculative decoding is disabled in the following cases:
377 - # 1. Prefill phase: Speculative decoding is not
378 - # used during the prefill phase.
379 - # 2. Auto-disable enabled: The running queue size exceeds
380 - # the specified threshold.
381 - # 3. No request: There are no requests in the batch.
382 - # In any of these cases, the proposer and scorer workers
383 - # are called normally.
384 - if num_lookahead_slots == 0 or len(
385 - execute_model_req.seq_group_metadata_list
386 - ) == 0 or disable_all_speculation:
401 + if no_spec:
402 return self._run_no_spec(execute_model_req,
403 skip_proposer=disable_all_speculation)
404 return self._run_speculative_decoding_step(execute_model_req,
@@ -464,8 +479,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
479 not called, meaning that the kv-cache in proposer for requests is not
480 updated, so they cannot enable spec decode in the rest decoding.
481 """
467 - if not skip_proposer:
468 - self.proposer_worker.execute_model(execute_model_req)
482
483 sampler_output = self.scorer_worker.execute_model(execute_model_req)
484 assert len(sampler_output) == 1
@@ -476,10 +489,20 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
489 if hidden_states is not None:
490 if self.previous_hidden_states is None:
491 self.previous_hidden_states = HiddenStates(
479 - execute_model_req.seq_group_metadata_list, hidden_states)
492 + hidden_states, execute_model_req.seq_group_metadata_list)
493 else:
494 self.previous_hidden_states.update(
482 - execute_model_req.seq_group_metadata_list, hidden_states)
495 + hidden_states, execute_model_req.seq_group_metadata_list)
496 +
497 + if not skip_proposer:
498 + # We prepare the prefill hidden states here so that there no
499 + # additional complexity in worker for spec_decode vs non_spec_decode
500 + # flow and execute_model doesn't need additional modifications.
501 + execute_model_req.previous_hidden_states = \
502 + prepare_prefill_hidden_states(
503 + sampler_output.prefill_hidden_states)
504 +
505 + self.proposer_worker.execute_model(execute_model_req)
506
507 sampler_output_to_return = (self._serialize_sampler_output_no_logprobs(
508 execute_model_req=execute_model_req, sampler_output=sampler_output)
@@ -507,15 +530,23 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
530 return False
531 num_lookahead_slots = data["num_lookahead_slots"]
532
510 - # Even if num_lookahead_slots is zero, we want to run the proposer model
511 - # as it may have KV.
512 - #
513 - # We run the proposer once per lookahead slot. In the future we should
514 - # delegate how many times it runs to the proposer.
515 - for _ in range(max(num_lookahead_slots, 1)):
516 - self.proposer_worker.execute_model()
533 + # In case of prefill, scorer_worker has to be run before proposer so
534 + # that the hidden states can be propagated to proposer when needed.
535 + if data["no_spec"]:
536 + self.scorer_worker.execute_model()
537 +
538 + if not data["disable_all_speculation"]:
539 + # Even if num_lookahead_slots is zero, we want to run the
540 + # proposer model as it may have KV.
541 + #
542 + # We run the proposer once per lookahead slot. In the future we
543 + # should delegate how many times it runs to the proposer.
544 + for _ in range(max(num_lookahead_slots, 1)):
545 + self.proposer_worker.execute_model()
546 +
547 + if not data["no_spec"]:
548 + self.scorer_worker.execute_model()
549
518 - self.scorer_worker.execute_model()
550 return True
551
552 @nvtx_range("spec_decode_worker._run_speculative_decoding_step")
@@ -546,6 +577,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
577 raise RuntimeError("Cannot handle cases where distributed draft "
578 "workers generate no tokens")
579
580 + execute_model_req.previous_hidden_states = None
581 +
582 with Timer() as scoring_timer:
583 proposal_scores = self.scorer.score_proposals(
584 execute_model_req,
@@ -651,10 +684,12 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
684 accepted_index = accepted_token_ids + 1 # Convert -1 to 0
685 accepted_index = accepted_index.count_nonzero(dim=1).add_(-1)
686 index = accepted_index[:, None, None].expand(-1, 1, hs_size)
687 + second_last_token_hidden_states = hidden_states[:, -2] # b x d
688 hidden_states = hidden_states.gather(1, index).squeeze(1) # b x d
689 # Store hidden states from target model for subsequent decode step
656 - self.previous_hidden_states = HiddenStates(seq_group_metadata_list,
657 - hidden_states)
690 + self.previous_hidden_states = HiddenStates(
691 + hidden_states, seq_group_metadata_list,
692 + second_last_token_hidden_states)
693
694 return accepted_token_ids, logprobs
695
@@ -951,3 +986,15 @@ def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
986 (proposer_cache_block_size_bytes + scorer_cache_block_size_bytes))
987
988 return new_num_gpu_blocks
989 +
990 +
991 +def prepare_prefill_hidden_states(
992 + prefill_hidden_states: torch.Tensor) -> HiddenStates:
993 + # For prefill step in proposer, we run the model for N-1 tokens
994 + # because Nth token will be processed in the first decode step. For
995 + # N-1 tokens, the input should be 0:N-1 hidden states which should
996 + # be concatanated with 1:N token (since output of scorer has to be
997 + # the input for proposer). Therefore, we shift the hidden states to
998 + # align n-1th hidden state with nth token.
999 + return HiddenStates(prefill_hidden_states.roll(
1000 + shifts=1, dims=0)) if prefill_hidden_states is not None else None
vllm/transformers_utils/config.py
+6 -4
@@ -11,10 +11,11 @@ from transformers.models.auto.modeling_auto import (
11 from vllm.envs import VLLM_USE_MODELSCOPE
12 from vllm.logger import init_logger
13 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
14 - InternVLChatConfig, JAISConfig,
15 - MedusaConfig, MLPSpeculatorConfig,
16 - MPTConfig, NemotronConfig,
17 - RWConfig, UltravoxConfig)
14 + EAGLEConfig, InternVLChatConfig,
15 + JAISConfig, MedusaConfig,
16 + MLPSpeculatorConfig, MPTConfig,
17 + NemotronConfig, RWConfig,
18 + UltravoxConfig)
19
20 if VLLM_USE_MODELSCOPE:
21 from modelscope import AutoConfig
@@ -32,6 +33,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
33 "jais": JAISConfig,
34 "mlp_speculator": MLPSpeculatorConfig,
35 "medusa": MedusaConfig,
36 + "eagle": EAGLEConfig,
37 "internvl_chat": InternVLChatConfig,
38 "nemotron": NemotronConfig,
39 "ultravox": UltravoxConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,5 +1,6 @@
1 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2 from vllm.transformers_utils.configs.dbrx import DbrxConfig
3 +from vllm.transformers_utils.configs.eagle import EAGLEConfig
4 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
5 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
6 # `FalconConfig` class from the official HuggingFace transformers library.
@@ -20,6 +21,7 @@ __all__ = [
21 "InternVLChatConfig",
22 "JAISConfig",
23 "MedusaConfig",
24 + "EAGLEConfig",
25 "MLPSpeculatorConfig",
26 "NemotronConfig",
27 "UltravoxConfig",
vllm/transformers_utils/configs/eagle.py new
+49
@@ -0,0 +1,49 @@
1 +import os
2 +from typing import Optional, Union
3 +
4 +from transformers import AutoConfig, PretrainedConfig
5 +
6 +
7 +class EAGLEConfig(PretrainedConfig):
8 + model_type = "eagle"
9 +
10 + def __init__(self,
11 + model: Union[PretrainedConfig, dict, None] = None,
12 + truncated_vocab_size: Optional[int] = None,
13 + **kwargs):
14 +
15 + model_config = None if model is None else (AutoConfig.for_model(
16 + **model) if isinstance(model, dict) else model)
17 +
18 + for k, v in kwargs.items():
19 + if k != "architectures" and k != "model_type" and hasattr(
20 + model_config, k):
21 + setattr(model_config, k, v)
22 +
23 + self.model = model_config
24 +
25 + if self.model is None:
26 + self.truncated_vocab_size = None
27 + else:
28 + self.truncated_vocab_size = self.model.vocab_size if \
29 + truncated_vocab_size is None else truncated_vocab_size
30 +
31 + if "architectures" not in kwargs:
32 + kwargs["architectures"] = ["EAGLEModel"]
33 +
34 + super().__init__(**kwargs)
35 +
36 + if self.model is not None:
37 + for k, v in self.model.to_dict().items():
38 + if not hasattr(self, k):
39 + setattr(self, k, v)
40 +
41 + @classmethod
42 + def from_pretrained(
43 + cls,
44 + pretrained_model_name_or_path: Union[str, os.PathLike],
45 + **kwargs,
46 + ) -> "EAGLEConfig":
47 + config_dict, kwargs = cls.get_config_dict(
48 + pretrained_model_name_or_path, **kwargs)
49 + return cls.from_dict(config_dict, **kwargs)
vllm/worker/model_runner.py
+34 -10
@@ -1,5 +1,6 @@
1 import dataclasses
2 import gc
3 +import inspect
4 import itertools
5 import time
6 import warnings
@@ -1192,6 +1193,18 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1193 max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
1194 input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1195 input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1196 +
1197 + # Prepare dummy previous_hidden_states only if needed by the model.
1198 + # This is used by draft models such as EAGLE.
1199 + previous_hidden_states = None
1200 + if "previous_hidden_states" in inspect.signature(
1201 + self.model.forward).parameters:
1202 + previous_hidden_states = torch.empty(
1203 + [max_batch_size,
1204 + self.model_config.get_hidden_size()],
1205 + dtype=self.model_config.dtype,
1206 + device=self.device)
1207 +
1208 intermediate_inputs = None
1209 if not get_pp_group().is_first_rank:
1210 intermediate_inputs = self.model.make_empty_intermediate_tensors(
@@ -1264,6 +1277,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1277 "stream":
1278 graph_capture_context.stream
1279 }
1280 + if previous_hidden_states is not None:
1281 + capture_inputs[
1282 + "previous_hidden_states"] = previous_hidden_states[:
1283 + batch_size]
1284 +
1285 if self.has_seqlen_agnostic:
1286 # Only used by Mamba-based models CUDA graph atm (Jamba)
1287 capture_inputs.update({
@@ -1462,6 +1480,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1480 if model_input.is_prompt:
1481 hidden_states = hidden_or_intermediate_states.index_select(
1482 0, indices)
1483 + output.prefill_hidden_states = hidden_or_intermediate_states
1484 elif decode_meta.use_cuda_graph:
1485 hidden_states = hidden_or_intermediate_states[:len(indices)]
1486 else:
@@ -1510,11 +1529,11 @@ class CUDAGraphRunner:
1529 # Note one iteration is not enough for torch.jit.script
1530 for _ in range(_NUM_WARMUP_ITERS):
1531 self.model(
1513 - input_ids,
1514 - positions,
1515 - kv_caches,
1516 - attn_metadata,
1517 - intermediate_inputs,
1532 + input_ids=input_ids,
1533 + positions=positions,
1534 + kv_caches=kv_caches,
1535 + attn_metadata=attn_metadata,
1536 + intermediate_tensors=intermediate_inputs,
1537 **kwargs,
1538 )
1539 torch.cuda.synchronize()
@@ -1523,11 +1542,11 @@ class CUDAGraphRunner:
1542 self._graph = torch.cuda.CUDAGraph()
1543 with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
1544 output_hidden_or_intermediate_states = self.model(
1526 - input_ids,
1527 - positions,
1528 - kv_caches,
1529 - attn_metadata,
1530 - intermediate_inputs,
1545 + input_ids=input_ids,
1546 + positions=positions,
1547 + kv_caches=kv_caches,
1548 + attn_metadata=attn_metadata,
1549 + intermediate_tensors=intermediate_inputs,
1550 **kwargs,
1551 )
1552 if hidden_or_intermediate_states is not None:
@@ -1588,6 +1607,11 @@ class CUDAGraphRunner:
1607 if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1608 self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1609 **kwargs)
1610 +
1611 + if "previous_hidden_states" in self.input_buffers:
1612 + self.input_buffers["previous_hidden_states"].copy_(
1613 + kwargs["previous_hidden_states"], non_blocking=True)
1614 +
1615 if intermediate_tensors is not None:
1616 for key in intermediate_tensors.tensors:
1617 if key != "model_execute_time" and key != "model_forward_time":
vllm/worker/multi_step_worker.py
+13 -8
@@ -1,5 +1,7 @@
1 from dataclasses import dataclass
2 -from typing import List, Optional, Tuple
2 +from typing import Dict, List, Optional, Tuple
3 +
4 +import torch
5
6 from vllm.distributed import broadcast_tensor_dict, get_pp_group
7 from vllm.sequence import ExecuteModelRequest, SamplerOutput
@@ -43,7 +45,7 @@ class MultiStepWorker(Worker):
45
46 def _get_driver_input_and_broadcast(
47 self, execute_model_req: ExecuteModelRequest
46 - ) -> Tuple[BroadcastableModelInput, WorkerInput]:
48 + ) -> Tuple[BroadcastableModelInput, WorkerInput, Dict[str, torch.Tensor]]:
49 """
50 Get the driver input and broadcast it to other workers.
51 """
@@ -85,7 +87,9 @@ class MultiStepWorker(Worker):
87 broadcast_data.update(model_input.as_broadcastable_tensor_dict())
88 broadcast_tensor_dict(broadcast_data, src=0)
89
88 - return model_input, worker_input
90 + # Retuning empty dict here to keep this compatible with
91 + # `LocalOrDistributedWorkerBase._get_driver_input_and_broadcast`
92 + return model_input, worker_input, {}
93
94 def _prepare_last_sampled_token_ids_for_tp_workers(
95 self,
@@ -130,7 +134,8 @@ class MultiStepWorker(Worker):
134 def prepare_input(
135 self,
136 execute_model_req: Optional[ExecuteModelRequest] = None,
133 - ) -> Optional[Tuple[StatefulModelInput, WorkerInput]]:
137 + ) -> Optional[Tuple[StatefulModelInput, WorkerInput, Dict[str,
138 + torch.Tensor]]]:
139 """
140 Depending on the current state of the request and multi step worker,
141 this method may skip the normal _prepare_model_input and
@@ -148,8 +153,8 @@ class MultiStepWorker(Worker):
153 return None
154
155 virtual_engine = execute_model_req.virtual_engine
151 - model_input, worker_input = self._get_driver_input_and_broadcast(
152 - execute_model_req)
156 + (model_input, worker_input,
157 + kwargs) = self._get_driver_input_and_broadcast(execute_model_req)
158 assert isinstance(model_input, StatefulModelInput)
159 if execute_model_req.is_first_multi_step:
160 # cache the worker input and model input for the next steps
@@ -162,7 +167,7 @@ class MultiStepWorker(Worker):
167 # loop
168 if broadcast_data is None:
169 return None
165 - model_input, worker_input = broadcast_data
170 + model_input, worker_input, kwargs = broadcast_data
171 assert isinstance(model_input, StatefulModelInput)
172 virtual_engine = worker_input.virtual_engine
173 if model_input.is_first_multi_step:
@@ -186,4 +191,4 @@ class MultiStepWorker(Worker):
191
192 assert model_input is not None
193 assert worker_input is not None
189 - return model_input, worker_input
194 + return model_input, worker_input, kwargs
vllm/worker/worker.py
+1 -1
@@ -86,7 +86,7 @@ class Worker(LocalOrDistributedWorkerBase):
86 or (speculative_config.draft_model_config.model ==
87 model_config.model) \
88 or (speculative_config.draft_model_config.hf_config.model_type
89 - not in ["medusa", "mlp_speculator"]) \
89 + not in ["medusa", "mlp_speculator", "eagle"]) \
90 else {"return_hidden_states": True}
91
92 ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
vllm/worker/worker_base.py
+50 -11
@@ -222,7 +222,9 @@ class LocalOrDistributedWorkerBase(WorkerBase):
222 raise NotImplementedError
223
224 def _get_worker_input_from_broadcast(
225 - self) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
225 + self
226 + ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput, Dict[
227 + str, torch.Tensor]]]:
228 """ Get the worker input from the broadcasted tensor dict. """
229 assert self.do_metadata_broadcast
230 assert not self.is_driver_worker
@@ -235,11 +237,13 @@ class LocalOrDistributedWorkerBase(WorkerBase):
237 self.model_runner.make_model_input_from_broadcasted_tensor_dict(
238 broadcast_data))
239
238 - return model_input, worker_input
240 + kwargs = extract_previous_hidden_states(broadcast_data)
241 +
242 + return model_input, worker_input, kwargs
243
244 def _get_driver_input_and_broadcast(
245 self, execute_model_req: ExecuteModelRequest
242 - ) -> Tuple[BroadcastableModelInput, WorkerInput]:
246 + ) -> Tuple[BroadcastableModelInput, WorkerInput, Dict[str, torch.Tensor]]:
247 """ Get the driver input and broadcast it to other workers. """
248 assert self.is_driver_worker
249
@@ -251,17 +255,21 @@ class LocalOrDistributedWorkerBase(WorkerBase):
255 execute_model_req.virtual_engine,
256 execute_model_req.finished_requests_ids))
257
258 + kwargs = extract_previous_hidden_states(execute_model_req)
259 +
260 if self.do_metadata_broadcast:
261 broadcast_data = worker_input.as_broadcastable_tensor_dict()
262 broadcast_data.update(model_input.as_broadcastable_tensor_dict())
263 + broadcast_data.update(kwargs)
264 broadcast_tensor_dict(broadcast_data, src=0)
265
259 - return model_input, worker_input
266 + return model_input, worker_input, kwargs
267
268 def prepare_input(
269 self,
270 execute_model_req: Optional[ExecuteModelRequest] = None
264 - ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
271 + ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput, Dict[
272 + str, torch.Tensor]]]:
273 """
274 Prepare the inputs to ModelRunner and workers.
275 """
@@ -291,7 +299,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
299 if inputs is None:
300 return None
301
294 - model_input, worker_input = inputs
302 + model_input, worker_input, kwargs = inputs
303 num_steps = worker_input.num_steps
304
305 self.execute_worker(worker_input)
@@ -312,9 +320,14 @@ class LocalOrDistributedWorkerBase(WorkerBase):
320 "model_execute_time", torch.tensor(0)).item()
321
322 output = self.model_runner.execute_model(
315 - model_input, self.kv_cache[worker_input.virtual_engine]
316 - if self.kv_cache is not None else None, intermediate_tensors,
317 - num_steps)
323 + model_input=model_input,
324 + kv_caches=self.kv_cache[worker_input.virtual_engine]
325 + if self.kv_cache is not None else None,
326 + intermediate_tensors=intermediate_tensors,
327 + num_steps=num_steps,
328 + **kwargs,
329 + )
330 +
331 model_execute_time = time.perf_counter() - start_time
332 if not get_pp_group().is_last_rank:
333 # output is IntermediateTensors
@@ -360,9 +373,15 @@ class LocalOrDistributedWorkerBase(WorkerBase):
373 if worker_input.num_seq_groups == 0:
374 return []
375
376 + kwargs = extract_previous_hidden_states(execute_model_req)
377 +
378 return self.model_runner.execute_model(
364 - model_input, self.kv_cache[worker_input.virtual_engine]
365 - if self.kv_cache is not None else None, intermediate_tensors)
379 + model_input=model_input,
380 + kv_caches=self.kv_cache[worker_input.virtual_engine]
381 + if self.kv_cache is not None else None,
382 + intermediate_tensors=intermediate_tensors,
383 + **kwargs,
384 + )
385
386
387 class WorkerWrapperBase:
@@ -439,3 +458,23 @@ class WorkerWrapperBase:
458 "This might cause deadlock in distributed execution.")
459 logger.exception(msg)
460 raise e
461 +
462 +
463 +def extract_previous_hidden_states(
464 + data: Union[ExecuteModelRequest, Dict[str, torch.Tensor]]) -> \
465 + Dict[str, torch.Tensor]:
466 + """If data contains previous_hidden_states, extract it. This returns a dict
467 + which can be used directly as additional kwargs in any following
468 + execute_model calls. This is used in draft models like EAGLE."""
469 + output = {}
470 +
471 + # When called from non-driver worker, data is dict but when called from
472 + # driver worker, data is ExecuteModelRequest.
473 + if isinstance(data, dict):
474 + if "previous_hidden_states" in data:
475 + output["previous_hidden_states"] = data["previous_hidden_states"]
476 + elif data.previous_hidden_states is not None:
477 + output["previous_hidden_states"] = data.previous_hidden_states\
478 + .hidden_states
479 +
480 + return output