A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Spec Decode] Introduce DraftModelRunner (#5799)
Cody Yu committed
Jun 28, 2024 at 09:17 UTC
b2c620230a6efdc590b06b10f8e89f42362a150a
15 files changed
+258
-37
tests/spec_decode/test_multi_step_worker.py
+3
@@ -7,6 +7,7 @@ import torch
7
8
from vllm.model_executor.utils import set_random_seed
9
from vllm.sequence import ExecuteModelRequest, Logprob, SamplerOutput
10
+from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
11
from vllm.spec_decode.multi_step_worker import MultiStepWorker
12
from vllm.spec_decode.top1_proposer import Top1Proposer
13
from vllm.worker.worker import Worker
@@ -85,6 +86,7 @@ def test_same_output_for_single_step():
86
block_size,
87
num_gpu_blocks,
88
seed,
89
+ model_runner_cls=TP1DraftModelRunner,
90
)
91
worker = create_worker(
92
Worker,
@@ -168,6 +170,7 @@ def test_same_output_for_multi_step():
170
block_size,
171
num_gpu_blocks,
172
seed,
173
+ model_runner_cls=TP1DraftModelRunner,
174
)
175
176
worker = create_worker(
tests/spec_decode/utils.py
+4
-1
@@ -14,6 +14,7 @@ from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
14
SequenceOutput)
15
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
16
from vllm.worker.cache_engine import CacheEngine
17
+from vllm.worker.model_runner import ModelRunner
18
from vllm.worker.worker import Worker
19
20
T = TypeVar("T", bound=Worker)
@@ -66,7 +67,8 @@ def create_worker(cls: Callable[..., T],
67
num_gpu_blocks: int,
68
seed: int,
69
is_driver_worker: bool = True,
69
- enforce_eager: bool = True) -> T:
70
+ enforce_eager: bool = True,
71
+ model_runner_cls: Optional[ModelRunner] = None) -> T:
72
engine_args = EngineArgs(
73
model=model_name,
74
seed=seed,
@@ -89,6 +91,7 @@ def create_worker(cls: Callable[..., T],
91
rank=0,
92
distributed_init_method=distributed_init_method,
93
is_driver_worker=is_driver_worker,
94
+ model_runner_cls=model_runner_cls,
95
)
96
97
worker.init_device()
vllm/sequence.py
+3
@@ -880,6 +880,8 @@ class ExecuteModelRequest:
880
running_queue_size: int = 0
881
# Optional hidden states from prior step.
882
previous_hidden_states: Optional[HiddenStates] = None
883
+ # The number of forward steps to run.
884
+ num_steps: int = 1
885
886
def clone(
887
self, seq_group_metadata_list: List[SequenceGroupMetadata]
@@ -893,4 +895,5 @@ class ExecuteModelRequest:
895
num_lookahead_slots=self.num_lookahead_slots,
896
running_queue_size=self.running_queue_size,
897
previous_hidden_states=self.previous_hidden_states,
898
+ num_steps=self.num_steps,
899
)
vllm/spec_decode/draft_model_runner.py
new
+170
@@ -0,0 +1,170 @@
1
+from typing import List, Optional
2
+
3
+import torch
4
+
5
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
+ ModelConfig, ParallelConfig, SchedulerConfig,
7
+ VisionLanguageConfig)
8
+from vllm.logger import init_logger
9
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
10
+from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
11
+ ModelRunner)
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+class TP1DraftModelRunner(ModelRunner):
17
+ """Specialized model runner for speculative decoding draft model.
18
+ Since the draft model always execute k forward passes consecutively to
19
+ generate k speculative tokens in a single speculative decoding step,
20
+ we could get rid of most CPU-GPU synchronization and data transfer
21
+ overheads by keeping model input and output tensors on GPU all the time.
22
+
23
+ This runner is still under development so there's no performance gain
24
+ at this moment. Currently we adopt a temporary solution that caches the
25
+ seq_group_metadata_list for multi-step execution, so that we can
26
+ leverage existing prepare_model_input to be compatible with the current
27
+ execution flow, but we plan to remove this cache and avoid calling
28
+ prepare_model_input in execute_model at all.
29
+
30
+ The detail development plan includes:
31
+ 1. Use "update_model_input" to update existing model_input without
32
+ creating a new one.
33
+ 2. Improve the performance of "update_model_input" with a GPU kernel.
34
+ 3. Support TP > 1 (this requires some designs because we do not expect
35
+ any broadcasting inside execute_model).
36
+ """
37
+
38
+ def __init__(
39
+ self,
40
+ model_config: ModelConfig,
41
+ parallel_config: ParallelConfig,
42
+ scheduler_config: SchedulerConfig,
43
+ device_config: DeviceConfig,
44
+ cache_config: CacheConfig,
45
+ load_config: LoadConfig,
46
+ lora_config: Optional[LoRAConfig],
47
+ kv_cache_dtype: Optional[str] = "auto",
48
+ is_driver_worker: bool = False,
49
+ vision_language_config: Optional[VisionLanguageConfig] = None,
50
+ return_hidden_states: bool = False,
51
+ ):
52
+ if return_hidden_states:
53
+ raise ValueError(
54
+ "return_hidden_states is not supported for TP1DraftModelRunner."
55
+ )
56
+
57
+ super().__init__(
58
+ model_config=model_config,
59
+ parallel_config=parallel_config,
60
+ scheduler_config=scheduler_config,
61
+ device_config=device_config,
62
+ cache_config=cache_config,
63
+ load_config=load_config,
64
+ lora_config=lora_config,
65
+ kv_cache_dtype=kv_cache_dtype,
66
+ is_driver_worker=is_driver_worker,
67
+ vision_language_config=vision_language_config,
68
+ return_hidden_states=return_hidden_states,
69
+ )
70
+
71
+ # TODO: Remove this cache when we are able to update model_input
72
+ # directly in advance_step.
73
+ self.cached_seq_group_metadata_list: Optional[
74
+ List[SequenceGroupMetadata]] = None
75
+
76
+ def prepare_model_input(
77
+ self,
78
+ seq_group_metadata_list: List[SequenceGroupMetadata],
79
+ ) -> ModelInputForGPUWithSamplingMetadata:
80
+ """A temporary solution that caches the seq_group_metadata_list
81
+ for multi-step execution.
82
+ TODO: In-place update model_input and remove this function.
83
+ """
84
+ self.cached_seq_group_metadata_list = seq_group_metadata_list
85
+ return super().prepare_model_input(seq_group_metadata_list)
86
+
87
+ def update_model_input(
88
+ self, model_input: ModelInputForGPUWithSamplingMetadata,
89
+ last_output: SamplerOutput
90
+ ) -> ModelInputForGPUWithSamplingMetadata:
91
+ """Prepare the model inputs for the next step.
92
+ TODO: In-place update model_input instead of calling
93
+ prepare_model_input.
94
+ """
95
+
96
+ # Append the output token to the sequence data.
97
+ assert self.cached_seq_group_metadata_list is not None
98
+ for seq_group_metadata, sequence_group_outputs in zip(
99
+ self.cached_seq_group_metadata_list, last_output.outputs):
100
+ seq_group_metadata.is_prompt = False
101
+
102
+ for seq_output in sequence_group_outputs.samples:
103
+ seq = seq_group_metadata.seq_data[seq_output.parent_seq_id]
104
+
105
+ token_id = seq_output.output_token
106
+ token_logprob = seq_output.logprobs[token_id]
107
+
108
+ seq.append_token_id(token_id, token_logprob.logprob)
109
+ seq.update_num_computed_tokens(1)
110
+
111
+ return self.prepare_model_input(self.cached_seq_group_metadata_list)
112
+
113
+ @torch.inference_mode()
114
+ def execute_model(
115
+ self,
116
+ model_input: ModelInputForGPUWithSamplingMetadata,
117
+ kv_caches: List[torch.Tensor],
118
+ num_steps: int = 1,
119
+ ) -> Optional[List[SamplerOutput]]:
120
+ # Since we do not broadcast data inside execute_model anymore,
121
+ # we need to figure out the best way to support TP > 1 in this
122
+ # case, because we will at least need to broadcast the sampled
123
+ # tokens to all workers.
124
+ if not self.is_driver_worker:
125
+ raise ValueError("TP1DraftModelRunner only supports TP=1.")
126
+
127
+ if self.lora_config:
128
+ assert model_input.lora_requests is not None
129
+ assert model_input.lora_mapping is not None
130
+ self.set_active_loras(model_input.lora_requests,
131
+ model_input.lora_mapping)
132
+
133
+ outputs: List[SamplerOutput] = []
134
+ for step in range(num_steps):
135
+ # Currently cuda graph is only supported by the decode phase.
136
+ assert model_input.attn_metadata is not None
137
+ prefill_meta = model_input.attn_metadata.prefill_metadata
138
+ decode_meta = model_input.attn_metadata.decode_metadata
139
+ if prefill_meta is None and decode_meta.use_cuda_graph:
140
+ assert model_input.input_tokens is not None
141
+ graph_batch_size = model_input.input_tokens.shape[0]
142
+ model_executable = self.graph_runners[graph_batch_size]
143
+ else:
144
+ model_executable = self.model
145
+
146
+ multi_modal_kwargs = model_input.multi_modal_kwargs or {}
147
+ hidden_states = model_executable(
148
+ input_ids=model_input.input_tokens,
149
+ positions=model_input.input_positions,
150
+ kv_caches=kv_caches,
151
+ attn_metadata=model_input.attn_metadata,
152
+ **multi_modal_kwargs,
153
+ )
154
+
155
+ # Compute the logits.
156
+ logits = self.model.compute_logits(hidden_states,
157
+ model_input.sampling_metadata)
158
+
159
+ # Sample the next token.
160
+ outputs.append(
161
+ self.model.sample(
162
+ logits=logits,
163
+ sampling_metadata=model_input.sampling_metadata,
164
+ ))
165
+
166
+ # Prepare the inputs for the next step.
167
+ if step != num_steps - 1:
168
+ model_input = self.update_model_input(model_input, outputs[-1])
169
+
170
+ return outputs
vllm/spec_decode/multi_step_worker.py
+16
-13
@@ -6,6 +6,7 @@ import torch
6
7
from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
8
SequenceGroupMetadata)
9
+from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
10
from vllm.spec_decode.interfaces import (SpeculativeProposals,
11
SpeculativeProposer)
12
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
@@ -67,22 +68,24 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
68
copied_execute_model_req = execute_model_req.clone(
69
copied_seq_group_metadata_list)
70
70
- # Assert enough KV space for sample_len tokens per sequence.
71
- self._assert_enough_kv_space(execute_model_req.seq_group_metadata_list,
72
- sample_len)
73
-
71
# Run model sample_len times.
72
model_outputs: List[SamplerOutput] = []
76
- for _ in range(sample_len):
77
- model_output: List[SamplerOutput] = super().execute_model(
73
+ if isinstance(self.model_runner, TP1DraftModelRunner):
74
+ copied_execute_model_req.num_steps = sample_len
75
+ model_outputs = self.execute_model(
76
execute_model_req=copied_execute_model_req)
79
- assert (len(model_output) == 1
80
- ), "composing multistep workers not supported"
81
- model_output = model_output[0]
82
-
83
- self._append_new_tokens(model_output,
84
- copied_seq_group_metadata_list)
85
- model_outputs.append(model_output)
77
+ else:
78
+ # TODO: Remove this branch once DraftModelRunner supports TP>1.
79
+ for _ in range(sample_len):
80
+ model_output: List[SamplerOutput] = super().execute_model(
81
+ execute_model_req=copied_execute_model_req)
82
+ assert (len(model_output) == 1
83
+ ), "composing multistep workers not supported"
84
+ model_output = model_output[0]
85
+
86
+ self._append_new_tokens(model_output,
87
+ copied_seq_group_metadata_list)
88
+ model_outputs.append(model_output)
89
90
return model_outputs, True
91
vllm/spec_decode/spec_decode_worker.py
+3
@@ -11,6 +11,7 @@ from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
11
HiddenStates, SamplerOutput, SequenceGroupMetadata,
12
get_all_seq_ids)
13
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
14
+from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
15
from vllm.spec_decode.interfaces import (SpeculativeProposals,
16
SpeculativeScorer, SpeculativeScores)
17
from vllm.spec_decode.metrics import AsyncMetricsCollector
@@ -117,6 +118,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
118
draft_tp = draft_parallel_config.tensor_parallel_size
119
target_tp = scorer_worker.parallel_config.tensor_parallel_size
120
121
+ if draft_tp == 1:
122
+ draft_worker_kwargs["model_runner_cls"] = TP1DraftModelRunner
123
proposer_worker = MultiStepWorker(**draft_worker_kwargs)
124
proposer_worker = SmallerTpProposerWorker.maybe_wrap_worker(
125
proposer_worker, draft_tp, target_tp)
vllm/worker/cpu_model_runner.py
+8
-3
@@ -351,7 +351,12 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
351
self,
352
model_input: CPUModelInput,
353
kv_caches: List[torch.Tensor],
354
- ) -> Optional[SamplerOutput]:
354
+ num_steps: int = 1,
355
+ ) -> Optional[List[SamplerOutput]]:
356
+ if num_steps > 1:
357
+ raise ValueError(
358
+ "CPU worker does not support multi-step execution.")
359
+
360
model_executable = self.model
361
execute_model_kwargs = {
362
"input_ids": model_input.input_tokens,
@@ -371,11 +376,11 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
376
377
# Only perform sampling in the driver worker.
378
if not self.is_driver_worker:
374
- return None
379
+ return []
380
381
# Sample the next token.
382
output = self.model.sample(
383
logits=logits,
384
sampling_metadata=model_input.sampling_metadata,
385
)
381
- return output
386
+ return [output]
vllm/worker/embedding_model_runner.py
+11
-4
@@ -57,7 +57,12 @@ class EmbeddingModelRunner(
57
self,
58
model_input: ModelInputForGPUWithPoolingMetadata,
59
kv_caches: List[torch.Tensor],
60
- ) -> Optional[PoolerOutput]:
60
+ num_steps: int = 1,
61
+ ) -> Optional[List[PoolerOutput]]:
62
+ if num_steps > 1:
63
+ raise ValueError(
64
+ "EmbeddingModelRunner does not support multi-step execution.")
65
+
66
if self.lora_config:
67
assert model_input.lora_requests is not None
68
assert model_input.lora_mapping is not None
@@ -91,10 +96,12 @@ class EmbeddingModelRunner(
96
97
# Only perform pooling in the driver worker.
98
if not self.is_driver_worker:
94
- return None
99
+ return []
100
96
- return self.model.pooler(hidden_states=hidden_states,
97
- pooling_metadata=model_input.pooling_metadata)
101
+ return [
102
+ self.model.pooler(hidden_states=hidden_states,
103
+ pooling_metadata=model_input.pooling_metadata)
104
+ ]
105
106
def make_model_input_from_broadcasted_tensor_dict(
107
self,
vllm/worker/model_runner.py
+7
-3
@@ -959,7 +959,11 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
959
self,
960
model_input: ModelInputForGPUWithSamplingMetadata,
961
kv_caches: List[torch.Tensor],
962
- ) -> SamplerOutput:
962
+ num_steps: int = 1,
963
+ ) -> Optional[List[SamplerOutput]]:
964
+ if num_steps > 1:
965
+ raise ValueError("num_steps > 1 is not supported in ModelRunner")
966
+
967
if self.lora_config:
968
assert model_input.lora_requests is not None
969
assert model_input.lora_mapping is not None
@@ -992,7 +996,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
996
997
# Only perform sampling in the driver worker.
998
if not self.is_driver_worker:
995
- return None
999
+ return []
1000
1001
# Sample the next token.
1002
output: SamplerOutput = self.model.sample(
@@ -1011,7 +1015,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1015
1016
output.hidden_states = hidden_states
1017
1014
- return output
1018
+ return [output]
1019
1020
1021
class CUDAGraphRunner:
vllm/worker/model_runner_base.py
+2
-1
@@ -150,7 +150,8 @@ class ModelRunnerBase(ABC, Generic[T]):
150
self,
151
model_input: T,
152
kv_caches: Optional[List[torch.Tensor]],
153
- ) -> Optional[SamplerOutput]:
153
+ num_steps: int = 1,
154
+ ) -> Optional[List[SamplerOutput]]:
155
"""
156
Execute the model on the given input.
157
"""
vllm/worker/neuron_model_runner.py
+7
-2
@@ -207,7 +207,12 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
207
self,
208
model_input: ModelInputForNeuron,
209
kv_caches: Optional[List[torch.Tensor]] = None,
210
- ) -> Optional[SamplerOutput]:
210
+ num_steps: int = 1,
211
+ ) -> Optional[List[SamplerOutput]]:
212
+ if num_steps > 1:
213
+ raise ValueError(
214
+ "NeuronModelRunner does not support multi-step execution.")
215
+
216
hidden_states = self.model(
217
input_ids=model_input.input_tokens,
218
positions=model_input.input_positions,
@@ -223,7 +228,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
228
logits=logits,
229
sampling_metadata=model_input.sampling_metadata,
230
)
226
- return output
231
+ return [output]
232
233
@property
234
def vocab_size(self) -> int:
vllm/worker/tpu_model_runner.py
+7
-2
@@ -444,7 +444,12 @@ class TPUModelRunner:
444
self,
445
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
446
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
447
- ) -> SamplerOutput:
447
+ num_steps: int = 1,
448
+ ) -> List[SamplerOutput]:
449
+ if num_steps > 1:
450
+ raise ValueError(
451
+ "TPUModelRunner does not support multi-step execution.")
452
+
453
assert seq_group_metadata_list is not None
454
assert len(seq_group_metadata_list) > 0
455
if seq_group_metadata_list[0].is_prompt:
@@ -462,7 +467,7 @@ class TPUModelRunner:
467
else:
468
sampler_outputs = self._execute_model(seq_group_metadata_list,
469
kv_caches)
465
- return SamplerOutput(sampler_outputs)
470
+ return [SamplerOutput(sampler_outputs)]
471
472
473
class ModelWrapper(nn.Module):
vllm/worker/worker.py
+4
-1
@@ -45,6 +45,7 @@ class Worker(LocalOrDistributedWorkerBase):
45
vision_language_config: Optional[VisionLanguageConfig] = None,
46
speculative_config: Optional[SpeculativeConfig] = None,
47
is_driver_worker: bool = False,
48
+ model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
49
) -> None:
50
self.model_config = model_config
51
self.parallel_config = parallel_config
@@ -78,7 +79,9 @@ class Worker(LocalOrDistributedWorkerBase):
79
"mlp_speculator") else {"return_hidden_states": True}
80
81
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
81
- if self.model_config.embedding_mode:
82
+ if model_runner_cls is not None:
83
+ ModelRunnerClass = model_runner_cls
84
+ elif self.model_config.embedding_mode:
85
ModelRunnerClass = EmbeddingModelRunner
86
self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
87
model_config,
vllm/worker/worker_base.py
+5
-4
@@ -228,11 +228,13 @@ class LocalOrDistributedWorkerBase(WorkerBase):
228
model_input: ModelRunnerInputBase = (
229
self.model_runner.prepare_model_input(
230
execute_model_req.seq_group_metadata_list))
231
+ num_steps = execute_model_req.num_steps
232
233
if self.do_metadata_broadcast:
234
broadcast_data = worker_input.as_broadcastable_tensor_dict()
235
broadcast_data.update(
236
model_input.as_broadcastable_tensor_dict())
237
+ broadcast_data["num_steps"] = num_steps
238
broadcast_tensor_dict(broadcast_data, src=0)
239
else:
240
assert self.do_metadata_broadcast
@@ -240,6 +242,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
242
if not broadcast_data:
243
return None
244
245
+ num_steps = broadcast_data.pop("num_steps")
246
worker_input = WorkerInput.from_broadcasted_tensor_dict(
247
broadcast_data)
248
model_input = (
@@ -252,10 +255,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
255
if worker_input.num_seq_groups == 0:
256
return []
257
255
- output = self.model_runner.execute_model(model_input, self.kv_cache)
256
- # Worker only supports single-step execution. Wrap the output in a
257
- # list to conform to interface.
258
- return [output]
258
+ return self.model_runner.execute_model(model_input, self.kv_cache,
259
+ num_steps)
260
261
262
class WorkerWrapperBase:
vllm/worker/xpu_model_runner.py
+8
-3
@@ -334,7 +334,12 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
334
self,
335
model_input: ModelInputForXPU,
336
kv_caches: List[torch.Tensor],
337
- ) -> Optional[SamplerOutput]:
337
+ num_steps: int = 1,
338
+ ) -> Optional[List[SamplerOutput]]:
339
+ if num_steps > 1:
340
+ raise ValueError(
341
+ "XPUModelRunner does not support multi-step execution.")
342
+
343
model_executable = self.model
344
execute_model_kwargs = {
345
"input_ids": model_input.input_tokens,
@@ -354,14 +359,14 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
359
360
# Only perform sampling in the driver worker.
361
if not self.is_driver_worker:
357
- return None
362
+ return []
363
364
# Sample the next token.
365
output = self.model.sample(
366
logits=logits,
367
sampling_metadata=model_input.sampling_metadata,
368
)
364
- return output
369
+ return [output]
370
371
def _prepare_prompt(
372
self,