A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[core] Multi Step Scheduling (#7000)
Co-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com>
William Lin committed
Aug 19, 2024 at 13:52 UTC
47b65a550866c7ffbd076ecb74106714838ce7da
13 files changed
+1004
-34
.buildkite/test-pipeline.yaml
+9
@@ -311,6 +311,15 @@ steps:
311
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
312
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
313
314
+- label: Multi-step Tests (4 GPUs) # 10min
315
+ working_dir: "/vllm-workspace/tests"
316
+ num_gpus: 4
317
+ source_file_dependencies:
318
+ - vllm/
319
+ - tests/multi_step/test_correctness.py
320
+ commands:
321
+ - pytest -v -s multi_step/test_correctness.py
322
+
323
- label: Pipeline Parallelism Test # 23min
324
working_dir: "/vllm-workspace/tests"
325
num_gpus: 4
tests/multi_step/__init__.py
tests/multi_step/test_correctness.py
new
+85
@@ -0,0 +1,85 @@
1
+# Test the AsyncLLMEngine with multi-step-decoding
2
+
3
+from typing import List
4
+
5
+import pytest
6
+
7
+from ..utils import RemoteOpenAIServer
8
+
9
+MODELS = [
10
+ "JackFram/llama-160m",
11
+]
12
+NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps
13
+NUM_PROMPTS = [10]
14
+
15
+DEFAULT_SERVER_ARGS: List[str] = [
16
+ "--disable-log-requests",
17
+ "--use-v2-block-manager",
18
+ "--worker-use-ray",
19
+ "--gpu-memory-utilization",
20
+ "0.85",
21
+ "--swap-space",
22
+ "16",
23
+]
24
+
25
+
26
+async def completions_with_server_args(prompts: List[str], model_name: str,
27
+ server_cli_args: List[str]):
28
+
29
+ outputs = None
30
+ with RemoteOpenAIServer(model_name, server_cli_args) as server:
31
+ client = server.get_async_client()
32
+ outputs = await client.completions.create(model=model_name,
33
+ prompt=prompts,
34
+ temperature=0,
35
+ stream=False,
36
+ max_tokens=5)
37
+ assert outputs is not None
38
+
39
+ return outputs
40
+
41
+
42
+@pytest.mark.parametrize("model", MODELS)
43
+@pytest.mark.parametrize(("tp_size, pp_size"), [
44
+ (1, 1),
45
+ (2, 2),
46
+])
47
+@pytest.mark.parametrize("eager_mode", [False, True])
48
+@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
49
+@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
50
+@pytest.mark.asyncio
51
+async def test_multi_step(example_prompts, model: str, tp_size: int,
52
+ pp_size: int, eager_mode: int,
53
+ num_scheduler_steps: int, num_prompts: int):
54
+
55
+ prompts = example_prompts
56
+ if len(prompts) < num_prompts:
57
+ prompts = prompts * ((num_prompts // len(prompts)) + 1)
58
+ prompts = prompts[:num_prompts]
59
+ assert len(prompts) == num_prompts
60
+
61
+ server_args = DEFAULT_SERVER_ARGS + ["--enforce-eager"]
62
+ ms_server_args = DEFAULT_SERVER_ARGS + \
63
+ ["--num-scheduler-steps", f"{num_scheduler_steps}"]
64
+
65
+ if eager_mode:
66
+ ms_server_args.append("--enforce-eager")
67
+
68
+ distributed_args = [
69
+ "--tensor-parallel-size",
70
+ str(tp_size),
71
+ "--pipeline-parallel-size",
72
+ str(pp_size),
73
+ ]
74
+
75
+ ref_completions = await completions_with_server_args(
76
+ prompts, model, server_args + distributed_args)
77
+ test_completions = await completions_with_server_args(
78
+ prompts, model, ms_server_args + distributed_args)
79
+
80
+ def get_text_generations(completions):
81
+ return [x.text for x in completions.choices]
82
+
83
+ ref_generations = get_text_generations(ref_completions)
84
+ test_generations = get_text_generations(test_completions)
85
+ assert ref_generations == test_generations
tests/worker/test_model_input.py
+77
@@ -10,6 +10,7 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
10
from vllm.worker.embedding_model_runner import (
11
ModelInputForGPUWithPoolingMetadata)
12
from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
13
+from vllm.worker.multi_step_model_runner import StatefulModelInput
14
15
16
class MockAttentionBackend(AttentionBackend):
@@ -154,3 +155,79 @@ def test_embedding_model_runner_input():
155
None) == getattr(attn_metadata, field.name, None)
156
# Pooling metadata is not broadcast.
157
assert received_model_input.pooling_metadata is None
158
+
159
+
160
+def test_multi_step_model_runner_input():
161
+ sampling_metadata = SamplingMetadata(
162
+ ["seq_group"],
163
+ "selected_token_indices",
164
+ "categorized_sample_indices",
165
+ "num_prompts",
166
+ )
167
+ attn_metadata = AttentionMetadata(
168
+ num_prefills=1,
169
+ num_prefill_tokens=2,
170
+ num_decode_tokens=3,
171
+ slot_mapping=torch.zeros(1),
172
+ )
173
+ frozen_model_input = ModelInputForGPUWithSamplingMetadata(
174
+ input_tokens=torch.ones(10),
175
+ input_positions=torch.ones(10),
176
+ sampling_metadata=sampling_metadata,
177
+ attn_metadata=attn_metadata)
178
+
179
+ model_input = StatefulModelInput(
180
+ frozen_model_input=frozen_model_input,
181
+ is_last_step=True,
182
+ is_first_multi_step=False,
183
+ current_step=4,
184
+ last_sampled_token_ids=torch.ones((10, 1)),
185
+ is_multi_step=True,
186
+ num_queries=8,
187
+ num_seqs=5,
188
+ cached_outputs=[],
189
+ )
190
+
191
+ assert isinstance(model_input, StatefulModelInput)
192
+
193
+ # Test round trip serialization.
194
+ tensor_dict = model_input.as_broadcastable_tensor_dict()
195
+ attn_backend = MockAttentionBackend()
196
+ received_model_input = (StatefulModelInput.from_broadcasted_tensor_dict(
197
+ tensor_dict, attn_backend=attn_backend))
198
+
199
+ receieved_frozen_input = received_model_input.frozen_model_input
200
+
201
+ # Check that received copy has correct values.
202
+ assert isinstance(received_model_input, StatefulModelInput)
203
+ assert receieved_frozen_input.input_tokens is not None
204
+ assert (receieved_frozen_input.input_tokens ==
205
+ frozen_model_input.input_tokens).all()
206
+ assert receieved_frozen_input.input_positions is not None
207
+ assert (receieved_frozen_input.input_positions ==
208
+ frozen_model_input.input_positions).all()
209
+ assert receieved_frozen_input.multi_modal_kwargs is None
210
+ assert (frozen_model_input.multi_modal_kwargs ==
211
+ frozen_model_input.multi_modal_kwargs)
212
+ assert receieved_frozen_input.lora_requests is None
213
+ assert (receieved_frozen_input.lora_requests ==
214
+ frozen_model_input.lora_requests)
215
+ assert receieved_frozen_input.lora_mapping is None
216
+ assert (
217
+ receieved_frozen_input.lora_mapping == frozen_model_input.lora_mapping)
218
+ for field in dataclasses.fields(AttentionMetadata):
219
+ assert getattr(receieved_frozen_input.attn_metadata, field.name,
220
+ None) == getattr(attn_metadata, field.name, None)
221
+ # For sampling metadata, only selected_token_indices is copied.
222
+ assert (receieved_frozen_input.sampling_metadata.selected_token_indices ==
223
+ sampling_metadata.selected_token_indices)
224
+ assert receieved_frozen_input.sampling_metadata.seq_groups is None
225
+
226
+ # check non frozen fields
227
+ assert received_model_input.is_last_step == model_input.is_last_step
228
+ assert (received_model_input.is_first_multi_step ==
229
+ model_input.is_first_multi_step)
230
+ assert received_model_input.current_step == model_input.current_step
231
+ assert (received_model_input.last_sampled_token_ids ==
232
+ model_input.last_sampled_token_ids).all()
233
+ assert received_model_input.is_multi_step == model_input.is_multi_step
vllm/engine/arg_utils.py
+6
-1
@@ -853,6 +853,12 @@ class EngineArgs:
853
"in low performance due to small KV cache space. Consider "
854
"setting --max-model-len to a smaller value.", max_model_len)
855
856
+ if self.num_scheduler_steps > 1 and not self.use_v2_block_manager:
857
+ self.use_v2_block_manager = True
858
+ logger.warning(
859
+ "Enabled BlockSpaceManagerV2 because it is "
860
+ "required for multi-step (--num-scheduler-steps > 1)")
861
+
862
speculative_config = SpeculativeConfig.maybe_create_spec_config(
863
target_model_config=model_config,
864
target_parallel_config=parallel_config,
@@ -881,7 +887,6 @@ class EngineArgs:
887
)
888
889
if self.num_scheduler_steps > 1:
884
- raise NotImplementedError("Multi-step is not yet supported.")
890
if speculative_config is not None:
891
raise ValueError("Speculative decoding is not supported with "
892
"multi-step (--num-scheduler-steps > 1)")
vllm/engine/async_llm_engine.py
+127
-8
@@ -1,9 +1,11 @@
1
import asyncio
2
import time
3
+from dataclasses import dataclass
4
from functools import partial
5
from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
6
Optional, Set, Tuple, Type, Union)
7
8
+import torch
9
from transformers import PreTrainedTokenizer
10
from typing_extensions import assert_never
11
@@ -27,7 +29,8 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
29
from vllm.pooling_params import PoolingParams
30
from vllm.prompt_adapter.request import PromptAdapterRequest
31
from vllm.sampling_params import SamplingParams
30
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
32
+from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
33
+ SequenceGroupMetadata)
34
from vllm.usage.usage_lib import UsageContext
35
from vllm.utils import print_warning_once
36
@@ -249,9 +252,25 @@ class RequestTracker:
252
return not self._new_requests.empty()
253
254
255
+@dataclass
256
+class SchedulerOutputState:
257
+ """Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
258
+ last_output: Optional[SamplerOutput] = None
259
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
260
+ scheduler_outputs: Optional[SchedulerOutputs] = None
261
+
262
+
263
class _AsyncLLMEngine(LLMEngine):
264
"""Extension of LLMEngine to add async methods."""
265
266
+ def __init__(self, *args, **kwargs):
267
+ super().__init__(*args, **kwargs)
268
+ pipeline_parallel_size = \
269
+ self.parallel_config.pipeline_parallel_size
270
+ self.cached_scheduler_outputs = [
271
+ SchedulerOutputState() for _ in range(pipeline_parallel_size)
272
+ ]
273
+
274
async def step_async(
275
self, virtual_engine: int
276
) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
@@ -264,13 +283,39 @@ class _AsyncLLMEngine(LLMEngine):
283
and updates the scheduler with the model outputs. Finally, it decodes
284
the sequences and returns the newly generated results.
285
"""
267
- seq_group_metadata_list, scheduler_outputs = self.scheduler[
268
- virtual_engine].schedule()
286
+ # these are cached outputs from previous iterations. None if on first
287
+ # iteration
288
+ cached_outputs = self.cached_scheduler_outputs[virtual_engine]
289
+ seq_group_metadata_list = cached_outputs.seq_group_metadata_list
290
+ scheduler_outputs = cached_outputs.scheduler_outputs
291
+ # skip the scheduler if there are any remaining steps in the seq groups.
292
+ # This ensures that the scheduler is only called again when the current
293
+ # batch has completed.
294
+ if not self._has_remaining_steps(seq_group_metadata_list):
295
+ seq_group_metadata_list, scheduler_outputs = self.scheduler[
296
+ virtual_engine].schedule()
297
+
298
+ if (self.scheduler_config.is_multi_step
299
+ and scheduler_outputs.num_lookahead_slots > 0):
300
+ # cache the scheduler outputs for the next iteration if we have
301
+ # lookahead slots
302
+ self._cache_scheduler_outputs_for_multi_step(
303
+ virtual_engine, seq_group_metadata_list, scheduler_outputs)
304
+
305
+ assert seq_group_metadata_list is not None
306
+ assert scheduler_outputs is not None
307
308
if not scheduler_outputs.is_empty():
271
- # Execute the model.
309
finished_requests_ids = self.scheduler[
310
virtual_engine].get_and_reset_finished_requests_ids()
311
+
312
+ # Check if we have a cached last_output from the previous iteration.
313
+ # For supporting PP this is probably the best way to pass the
314
+ # sampled_token_ids, as a separate broadcast over all the PP stages
315
+ # will cause one virtual engine's microbatch to block the pipeline.
316
+ last_sampled_token_ids = \
317
+ self._get_last_sampled_token_ids(virtual_engine)
318
+
319
execute_model_req = ExecuteModelRequest(
320
seq_group_metadata_list=seq_group_metadata_list,
321
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
@@ -279,15 +324,35 @@ class _AsyncLLMEngine(LLMEngine):
324
virtual_engine=virtual_engine,
325
num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
326
running_queue_size=scheduler_outputs.running_queue_size,
282
- finished_requests_ids=finished_requests_ids)
327
+ finished_requests_ids=finished_requests_ids,
328
+ # We use ExecuteModelRequest to pass the last sampled_token_ids
329
+ # to each of the non-last PP stages for in-place prepare_input.
330
+ last_sampled_token_ids=last_sampled_token_ids)
331
+ # Execute the model.
332
output = await self.model_executor.execute_model_async(
333
execute_model_req)
334
+ # we need to do this here so that last step's sampled_token_ids can
335
+ # be passed to the next iteration for PP.
336
+ if self.scheduler_config.is_multi_step:
337
+ self._update_cached_scheduler_output(virtual_engine, output)
338
else:
339
output = []
340
288
- request_outputs = self._process_model_outputs(
289
- output, scheduler_outputs.scheduled_seq_groups,
290
- scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
341
+ # Finish the current step for all the sequence groups.
342
+ if self.scheduler_config.is_multi_step:
343
+ for seq_group in seq_group_metadata_list:
344
+ seq_group.finish_step()
345
+
346
+ if not self._has_remaining_steps(seq_group_metadata_list):
347
+ # clear the cache if we have finished all the steps
348
+ if self.scheduler_config.is_multi_step:
349
+ self.cached_scheduler_outputs[
350
+ virtual_engine] = SchedulerOutputState()
351
+ request_outputs = self._process_model_outputs(
352
+ output, scheduler_outputs.scheduled_seq_groups,
353
+ scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
354
+ else:
355
+ request_outputs = []
356
357
# Log stats.
358
self.do_log_stats(scheduler_outputs, output)
@@ -297,6 +362,60 @@ class _AsyncLLMEngine(LLMEngine):
362
363
return request_outputs
364
365
+ def _has_remaining_steps(
366
+ self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
367
+ ) -> bool:
368
+ if (not self.scheduler_config.is_multi_step
369
+ or not seq_group_metadata_list):
370
+ return False
371
+
372
+ # TODO(will) this is a sanity check for nowto make sure that all the
373
+ # seqs are on the same steps. Eventually we will want to do some sort of
374
+ # dynamic scheduling when doing multi-step decoding.
375
+ ref_remaining_steps = seq_group_metadata_list[0].state.remaining_steps
376
+ if any([
377
+ seq_group.state.remaining_steps != ref_remaining_steps
378
+ for seq_group in seq_group_metadata_list[1:]
379
+ ]):
380
+ raise AssertionError(("All running sequence groups should "
381
+ "have the same remaining steps."))
382
+
383
+ return ref_remaining_steps > 0
384
+
385
+ def _cache_scheduler_outputs_for_multi_step(
386
+ self, virtual_engine: int,
387
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
388
+ scheduler_outputs: SchedulerOutputs) -> None:
389
+ self.cached_scheduler_outputs[
390
+ virtual_engine].seq_group_metadata_list = seq_group_metadata_list
391
+ self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
392
+ scheduler_outputs
393
+ self.cached_scheduler_outputs[virtual_engine].last_output = None
394
+
395
+ def _get_last_sampled_token_ids(
396
+ self, virtual_engine: int) -> Optional[torch.Tensor]:
397
+ cached_last_output = self.cached_scheduler_outputs[
398
+ virtual_engine].last_output
399
+ if (self.scheduler_config.is_multi_step
400
+ and self.parallel_config.pipeline_parallel_size > 1
401
+ and cached_last_output is not None
402
+ and cached_last_output.sampled_token_ids_cpu is not None):
403
+ return cached_last_output.sampled_token_ids_cpu
404
+ return None
405
+
406
+ def _update_cached_scheduler_output(
407
+ self, virtual_engine: int,
408
+ output: List[Optional[SamplerOutput]]) -> None:
409
+ if (self.parallel_config.pipeline_parallel_size > 1 and len(output) > 0
410
+ and output[0] is not None):
411
+ last_output = output[-1]
412
+ assert last_output is not None
413
+ assert last_output.sampled_token_ids_cpu is not None
414
+ assert last_output.sampled_token_ids is None
415
+ assert last_output.sampled_token_probs is None
416
+ self.cached_scheduler_outputs[
417
+ virtual_engine].last_output = last_output
418
+
419
async def stop_remote_worker_execution_loop_async(self) -> None:
420
"""Stop the remote worker execution loop."""
421
await self.model_executor.stop_remote_worker_execution_loop_async()
vllm/executor/gpu_executor.py
+10
-4
@@ -69,13 +69,19 @@ class GPUExecutor(ExecutorBase):
69
distributed_init_method: Optional[str] = None) -> Dict:
70
worker_kwargs = self._get_worker_kwargs(local_rank, rank,
71
distributed_init_method)
72
- if self.speculative_config is None:
73
- worker_kwargs.update(worker_module_name="vllm.worker.worker",
74
- worker_class_name="Worker")
75
- else:
72
+
73
+ if self.scheduler_config.is_multi_step:
74
+ worker_kwargs.update(
75
+ worker_module_name="vllm.worker.multi_step_worker",
76
+ worker_class_name="MultiStepWorker")
77
+ elif self.speculative_config:
78
worker_kwargs.update(
79
worker_module_name="vllm.spec_decode.spec_decode_worker",
80
worker_class_name="create_spec_worker")
81
+ else:
82
+ worker_kwargs.update(worker_module_name="vllm.worker.worker",
83
+ worker_class_name="Worker")
84
+
85
return worker_kwargs
86
87
def _create_worker(self,
vllm/executor/ray_gpu_executor.py
+3
@@ -94,6 +94,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
94
if self.speculative_config is not None:
95
worker_module_name = "vllm.spec_decode.spec_decode_worker"
96
worker_class_name = "create_spec_worker"
97
+ elif self.scheduler_config.is_multi_step:
98
+ worker_module_name = "vllm.worker.multi_step_worker"
99
+ worker_class_name = "MultiStepWorker"
100
else:
101
worker_module_name = "vllm.worker.worker"
102
worker_class_name = "Worker"
vllm/sequence.py
+5
-5
@@ -9,7 +9,6 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
9
Tuple, Union, cast)
10
11
import msgspec
12
-import numpy
12
import torch
13
14
from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
@@ -1082,7 +1081,10 @@ class SamplerOutput(
1081
1082
# On-device tensor containing the sampled token ids.
1083
sampled_token_ids: Optional[torch.Tensor] = None
1085
- sampled_token_ids_numpy: Optional[numpy.ndarray] = None
1084
+ # CPU tensor containing the sampled token ids. Used during multi-step to
1085
+ # return the sampled token ids from last rank to AsyncLLMEngine to be
1086
+ # 'broadcasted' to all other PP ranks for next step.
1087
+ sampled_token_ids_cpu: Optional[torch.Tensor] = None
1088
1089
# Spec decode metrics populated by workers.
1090
spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
@@ -1257,9 +1259,7 @@ class ExecuteModelRequest(
1259
assert len(self.seq_group_metadata_list) > 0
1260
first_seq_group = self.seq_group_metadata_list[0]
1261
assert first_seq_group.state is not None
1260
- num_steps = first_seq_group.state.num_steps
1261
- current_step = first_seq_group.state.current_step
1262
- return num_steps - current_step == 1
1262
+ return first_seq_group.state.remaining_steps == 1
1263
1264
@property
1265
def current_step(self) -> int:
vllm/worker/model_runner_base.py
+34
-12
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
14
from vllm.attention.backends.abstract import AttentionBackend
15
from vllm.model_executor import SamplingMetadata
16
17
-T = TypeVar('T', bound="ModelRunnerInputBase")
17
+T = TypeVar('T', bound="BroadcastableModelInput")
18
19
20
def _add_attn_metadata_broadcastable_dict(
@@ -81,18 +81,26 @@ def _add_sampling_metadata_broadcastable_dict(
81
sampling_metadata.selected_token_indices)
82
83
84
-@dataclasses.dataclass(frozen=True)
85
-class ModelRunnerInputBase(ABC):
86
- """Local inputs to each worker's model runner. May contain
87
- device-specific data. Different worker backends may have different methods
88
- of converting from the global ExecuteModelRequest produced by the LLM
89
- engine to the worker-local ModelRunnerInputBase objects.
90
-
91
- Model runners that support multi-GPU execution should define a
92
- ModelRunnerInputBase subclass, add their required fields, and specify how to
93
- serialize/deserialize a ModelInput for broadcast between workers.
84
+def _init_frozen_model_input_from_tensor_dict(
85
+ frozen_model_input_cls: Type["ModelRunnerInputBase"],
86
+ tensor_dict: Dict[str, Any]) -> Dict[str, Any]:
87
"""
88
+ Helper method to initialize a frozen ModelInput based on broadcastable
89
+ """
90
+ valid_tensor_kwargs = {}
91
+ for field in dataclasses.fields(frozen_model_input_cls):
92
+ val = tensor_dict.pop(field.name, None)
93
+ if val is not None:
94
+ valid_tensor_kwargs[field.name] = val
95
+
96
+ frozen_model_input = frozen_model_input_cls(**valid_tensor_kwargs)
97
+ tensor_dict["frozen_model_input"] = frozen_model_input
98
+ return tensor_dict
99
100
+
101
+class BroadcastableModelInput(ABC):
102
+
103
+ @abstractmethod
104
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
105
"""
106
Extract broadcastable fields. Override for fields that require some
@@ -109,11 +117,25 @@ class ModelRunnerInputBase(ABC):
117
) -> T:
118
"""
119
Pop fields from the given tensor_dict and populate a new instance of
112
- ModelRunnerInputBase.
120
+ BroadcastableModelInput.
121
"""
122
raise NotImplementedError
123
124
125
+@dataclasses.dataclass(frozen=True)
126
+class ModelRunnerInputBase(BroadcastableModelInput):
127
+ """Local inputs to each worker's model runner. May contain
128
+ device-specific data. Different worker backends may have different methods
129
+ of converting from the global ExecuteModelRequest produced by the LLM
130
+ engine to the worker-local ModelRunnerInputBase objects.
131
+
132
+ Model runners that support multi-GPU execution should define a
133
+ ModelRunnerInputBase subclass, add their required fields, and specify how to
134
+ serialize/deserialize a ModelInput for broadcast between workers.
135
+ """
136
+ pass
137
+
138
+
139
class ModelRunnerInputBuilderBase(ABC, Generic[T]):
140
"""A builder to create ModelRunnerInputBase objects.
141
"""
vllm/worker/multi_step_model_runner.py
new
+453
@@ -0,0 +1,453 @@
1
+from dataclasses import dataclass, field
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
3
+
4
+try:
5
+ from vllm.attention.backends.flash_attn import FlashAttentionMetadata
6
+except ModuleNotFoundError:
7
+ # vllm_flash_attn is not installed, use the identical ROCm FA metadata
8
+ from vllm.attention.backends.rocm_flash_attn import (
9
+ ROCmFlashAttentionMetadata as FlashAttentionMetadata)
10
+
11
+import torch
12
+
13
+from vllm import _custom_ops as ops
14
+from vllm.distributed import get_pp_group
15
+from vllm.logger import init_logger
16
+from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
17
+ Logprob, SamplerOutput, SequenceGroupMetadata,
18
+ SequenceOutput)
19
+from vllm.worker.model_runner import (GPUModelRunnerBase,
20
+ ModelInputForGPUWithSamplingMetadata)
21
+from vllm.worker.model_runner_base import (
22
+ BroadcastableModelInput, _init_attn_metadata_from_tensor_dict,
23
+ _init_frozen_model_input_from_tensor_dict,
24
+ _init_sampling_metadata_from_tensor_dict)
25
+
26
+from ..model_executor.model_loader.tensorizer import TensorizerConfig
27
+
28
+if TYPE_CHECKING:
29
+ from vllm.attention.backends.abstract import AttentionBackend
30
+
31
+logger = init_logger(__name__)
32
+
33
+
34
+@dataclass
35
+class ModelOutput:
36
+ """The output of a single model forward pass.
37
+
38
+ The sampler_output_ready_event is set when the tensors in
39
+ sampler_output are ready (the model+sampler forward pass has
40
+ completed). We use the event to synchronize the GPU->CPU transfer,
41
+ which we want to only run when the data has been written to the
42
+ GPU tensors. Until the event is ready, the tensors in sampler_output
43
+ will have garbage data.
44
+
45
+ There are two scenarios:
46
+ 1. The output tensors are ready and we can pythonize them immediately.
47
+ 2. The output tensors are not ready and we need to wait for the event to be
48
+ ready.
49
+ """
50
+ sampler_output: SamplerOutput
51
+ sampler_output_ready_event: torch.cuda.Event
52
+ sampled_token_ids: Optional[torch.Tensor] = None
53
+ pythonized: bool = False
54
+
55
+ def pythonize(self, input_metadata: "StatefulModelInput",
56
+ copy_stream: torch.cuda.Stream,
57
+ pinned_sampled_token_buffer: torch.Tensor) -> None:
58
+ """Pythonize the output. Blocking."""
59
+ if not self.pythonized:
60
+ self._pythonize_sampler_output(input_metadata, copy_stream,
61
+ pinned_sampled_token_buffer, True)
62
+ self.pythonized = True
63
+
64
+ def maybe_pythonize(self, input_metadata: "StatefulModelInput",
65
+ copy_stream: torch.cuda.Stream,
66
+ pinned_sampled_token_buffer: torch.Tensor) -> None:
67
+ """Pythonize the output if ready, else return None. Non-blocking."""
68
+ if not self.pythonized:
69
+ self.pythonized = self._pythonize_sampler_output(
70
+ input_metadata, copy_stream, pinned_sampled_token_buffer,
71
+ False)
72
+
73
+ def _pythonize_sampler_output(self, input_metadata: "StatefulModelInput",
74
+ copy_stream: torch.cuda.Stream,
75
+ pinned_sampled_token_buffer: torch.Tensor,
76
+ blocking: bool) -> bool:
77
+ """
78
+ If blocking is set, will block until the forward pass for the output is
79
+ ready and pythonize the output.
80
+ """
81
+ assert self.sampled_token_ids is not None
82
+ if not blocking and not self.sampler_output_ready_event.query():
83
+ return False
84
+
85
+ if blocking:
86
+ self.sampler_output_ready_event.synchronize()
87
+ with torch.cuda.stream(copy_stream):
88
+ _pythonize_sampler_output(input_metadata, self.sampler_output,
89
+ pinned_sampled_token_buffer,
90
+ self.sampled_token_ids)
91
+ return True
92
+
93
+
94
+@dataclass(frozen=False)
95
+class StatefulModelInput(BroadcastableModelInput):
96
+ # actual frozen model input dataclass passed to _base_model_runner
97
+ frozen_model_input: Optional[ModelInputForGPUWithSamplingMetadata] = None
98
+
99
+ # list of model outputs for each step, may not be all pythonized
100
+ cached_outputs: List[ModelOutput] = field(default_factory=list)
101
+
102
+ # used to pass sampled token ids from the last step to the current step for
103
+ # TP workers. Used to append to end of outputs and used by advance_step
104
+ last_sampled_token_ids: Optional[torch.Tensor] = None
105
+ current_step: int = 0
106
+ is_multi_step: bool = True
107
+ is_last_step: bool = False
108
+ is_first_multi_step: bool = False
109
+ # ping-pong data structures for multi-step to wait on the previous step
110
+ step_cuda_events: List[torch.cuda.Event] = field(
111
+ default_factory=lambda: [torch.cuda.Event(blocking=True)] * 2)
112
+ num_seqs: int = -1
113
+ num_queries: int = -1
114
+
115
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
116
+ assert self.frozen_model_input is not None
117
+ tensor_dict = self.frozen_model_input.as_broadcastable_tensor_dict()
118
+ new_tensor_dict = {
119
+ 'last_sampled_token_ids': self.last_sampled_token_ids,
120
+ 'current_step': self.current_step,
121
+ 'is_multi_step': self.is_multi_step,
122
+ 'is_last_step': self.is_last_step,
123
+ 'is_first_multi_step': self.is_first_multi_step,
124
+ 'num_seqs': self.num_seqs,
125
+ 'num_queries': self.num_queries,
126
+ }
127
+ tensor_dict.update(new_tensor_dict)
128
+ return tensor_dict
129
+
130
+ @classmethod
131
+ def from_broadcasted_tensor_dict(
132
+ cls,
133
+ tensor_dict: Dict[str, Any],
134
+ attn_backend: Optional["AttentionBackend"] = None,
135
+ ) -> "StatefulModelInput":
136
+ tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
137
+ if attn_backend is not None:
138
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
139
+ attn_backend, tensor_dict)
140
+ tensor_dict = _init_frozen_model_input_from_tensor_dict(
141
+ ModelInputForGPUWithSamplingMetadata, tensor_dict)
142
+
143
+ return cls(**tensor_dict)
144
+
145
+ def record_step_event(self, current_stream: torch.cuda.Stream):
146
+ # record the event for the current step so that the next step can sync
147
+ # on it. We modulo by 2 to keep the events in a circular buffer and
148
+ # support any attn backends that may be supported in the future. ie
149
+ # Flashinfer would want two DecodeWrappers to overlap the CPU and GPU.
150
+ self.step_cuda_events[self.current_step & 1] = \
151
+ torch.cuda.Event(blocking=True)
152
+ self.step_cuda_events[self.current_step & 1].record(current_stream)
153
+
154
+ def wait_previous_step(self):
155
+ # These cuda events are an explicit synchronization to ensure that
156
+ # advance_step() (for other attn backends that may be supported in the
157
+ # future) do not clobber any data structures that is also used by any
158
+ # enqueued forwards steps. For distributed case, only a single event is
159
+ # needed, but for single GPU case, since we can let the CPU run much
160
+ # further ahead, two events allow us to overlap the advance_step with
161
+ # the previous forward (ie using two DecodeWrappers for flashinfer
162
+ # backend)
163
+ self.step_cuda_events[(self.current_step + 1) & 1].wait()
164
+
165
+ def add_sampler_output(self,
166
+ sampler_output: SamplerOutput,
167
+ sampled_token_ids: Optional[torch.Tensor] = None):
168
+ self.cached_outputs.append(
169
+ ModelOutput(sampler_output=sampler_output,
170
+ sampler_output_ready_event=None,
171
+ sampled_token_ids=sampled_token_ids,
172
+ pythonized=False))
173
+
174
+
175
+# MutableModelInputForGPUWithMultiStepMetadata is not subclass of
176
+# ModelInputForGPU but it wraps the actual input dataclass and adds multi-step
177
+# metadata
178
+# mypy: disable-error-code=type-var
179
+class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
180
+ # mypy: enable-error-code=type-var
181
+
182
+ def __init__(self, base_model_runner: GPUModelRunnerBase, *args, **kwargs):
183
+ super().__init__(*args, **kwargs)
184
+
185
+ # uses the base model runner to execute the model and wraps it with
186
+ # multi-step logic
187
+ self._base_model_runner: GPUModelRunnerBase = base_model_runner
188
+
189
+ self.is_multi_step = self.scheduler_config.is_multi_step
190
+ # used to copy tensors from GPU to CPU asynchronously
191
+ self._copy_stream = torch.cuda.Stream()
192
+ self.pinned_sampled_token_ids: Optional[torch.Tensor] = None
193
+
194
+ def make_model_input_from_broadcasted_tensor_dict(
195
+ self, tensor_dict: Dict[str, Any]) -> StatefulModelInput:
196
+ model_input = (StatefulModelInput.from_broadcasted_tensor_dict(
197
+ tensor_dict,
198
+ attn_backend=self.attn_backend,
199
+ ))
200
+ return model_input
201
+
202
+ def prepare_model_input(
203
+ self,
204
+ seq_group_metadata_list: List[SequenceGroupMetadata],
205
+ virtual_engine: int = 0,
206
+ finished_requests_ids: Optional[List[str]] = None
207
+ ) -> StatefulModelInput:
208
+ frozen_model_input = self._base_model_runner.prepare_model_input(
209
+ seq_group_metadata_list, virtual_engine, finished_requests_ids)
210
+
211
+ model_input = StatefulModelInput(
212
+ frozen_model_input=frozen_model_input,
213
+ num_seqs=len(frozen_model_input.seq_lens),
214
+ num_queries=len(frozen_model_input.query_lens),
215
+ )
216
+ return model_input
217
+
218
+ @torch.inference_mode()
219
+ def execute_model(
220
+ self,
221
+ model_input: StatefulModelInput,
222
+ kv_caches: List[torch.Tensor],
223
+ intermediate_tensors: Optional[IntermediateTensors] = None,
224
+ num_steps: int = 1,
225
+ ) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]:
226
+ """
227
+ Execute the model for a single step and update multi-step
228
+ metadata
229
+ """
230
+ assert num_steps == 1, "MultiStepModelRunner only supports num_steps=1"
231
+ frozen_model_input = model_input.frozen_model_input
232
+ assert frozen_model_input is not None
233
+
234
+ # path for warm up runs
235
+ if not model_input.is_multi_step:
236
+ return self._base_model_runner.execute_model(
237
+ frozen_model_input, kv_caches, intermediate_tensors, num_steps)
238
+
239
+ # make sure we skip the sampler on the lask rank and only pythonize
240
+ # if CPU is ahead.
241
+ if self.is_driver_worker and get_pp_group().is_last_rank:
242
+ if self.pinned_sampled_token_ids is None:
243
+ self.pinned_sampled_token_ids = torch.zeros(
244
+ (self.scheduler_config.max_num_seqs, 1),
245
+ dtype=torch.long,
246
+ device="cpu",
247
+ pin_memory=True)
248
+
249
+ self._base_model_runner.model.sampler.include_gpu_probs_tensor = (
250
+ True)
251
+ if frozen_model_input.sampling_metadata:
252
+ frozen_model_input.sampling_metadata.skip_sampler_cpu_output = (
253
+ True)
254
+
255
+ # some pre-execute model logic for multi-step:
256
+ # - if it's the first step, we need to reset the sampling tensors
257
+ # - if it's not the first step, we need to advance the step using the
258
+ # appended sampler output from last iteration
259
+ # - also maybe pythonize if CPU is ahead of GPU
260
+
261
+ current_stream = torch.cuda.current_stream()
262
+ if not model_input.is_first_multi_step:
263
+ # Explicitly block on the previous step's forward to make sure we
264
+ # don't clobber any GPU tensors still in use.
265
+ # This is not needed for flashattn backend, but for other attn
266
+ # backends such as flashinfer that performs extra CPU operations on
267
+ # input metadata we may need to synchronize any CPU operations that
268
+ # might clobber enqueued forwards. (prevents CPU from running too
269
+ # far ahead if needed)
270
+ model_input.wait_previous_step()
271
+ model_input = self._advance_step(
272
+ model_input, model_input.cached_outputs[-1].sampler_output)
273
+
274
+ # Execute the model
275
+ output = self._base_model_runner.execute_model(frozen_model_input,
276
+ kv_caches,
277
+ intermediate_tensors,
278
+ num_steps=1)
279
+
280
+ # record the event for the current step so that the next step can sync
281
+ model_input.record_step_event(current_stream)
282
+
283
+ if get_pp_group().is_last_rank and self.is_driver_worker:
284
+ assert len(
285
+ output
286
+ ) == 1, "MultiStepModelRunner requires single-step base_models"
287
+
288
+ # event for the pythonization so that we only pythonize if the
289
+ # tensors are ready. May be able to be combined with the step event
290
+ output_ready_event = torch.cuda.Event()
291
+ output_ready_event.record(current_stream)
292
+ if self.parallel_config.pipeline_parallel_size > 1:
293
+ output[0].sampled_token_ids_cpu = output[
294
+ 0].sampled_token_ids.cpu()
295
+ model_input.cached_outputs.append(
296
+ ModelOutput(output[0], output_ready_event,
297
+ output[0].sampled_token_ids, False))
298
+ # make sure we dont try to serialize any GPU tensors
299
+ output[0].sampled_token_ids = None
300
+ output[0].sampled_token_probs = None
301
+ output[0].logprobs = None
302
+ # Pythonize the output if CPU is ahead and the previous step is
303
+ # ready.
304
+ for model_output in model_input.cached_outputs:
305
+ model_output.maybe_pythonize(model_input, self._copy_stream,
306
+ self.pinned_sampled_token_ids)
307
+
308
+ model_input.current_step += 1
309
+
310
+ if not get_pp_group().is_last_rank:
311
+ # Should be IntermediateTensors
312
+ assert isinstance(output, IntermediateTensors)
313
+ return output
314
+ if not self.is_driver_worker:
315
+ return []
316
+
317
+ # Pythonize the output and block if needed since it is the last step
318
+ if model_input.is_last_step:
319
+ outputs = []
320
+ for output in model_input.cached_outputs:
321
+ output.pythonize(model_input, self._copy_stream,
322
+ self.pinned_sampled_token_ids)
323
+ outputs.append(output.sampler_output)
324
+ return outputs
325
+
326
+ # should be [SamplerOutput]
327
+ return output
328
+
329
+ def _update_sampling_metadata(self, sampling_metadata, num_seqs,
330
+ num_queries):
331
+
332
+ assert sampling_metadata.num_prompts == 0
333
+ assert len(sampling_metadata.seq_groups) == num_queries
334
+ assert sampling_metadata.selected_token_indices.shape == (
335
+ num_queries, )
336
+ # assert sampling_metadata.categorized_sample_indices == TODO: Add if needed # noqa: E501
337
+
338
+ # Verify that all sequences are decodes
339
+ for i in range(num_queries):
340
+ seq_group = sampling_metadata.seq_groups[i]
341
+
342
+ assert seq_group.is_prompt is False # No prompt
343
+ assert seq_group.prompt_logprob_indices == [] # No prompt
344
+ assert seq_group.sample_indices == [i] # Simple
345
+ assert seq_group.seq_len is None # Decode
346
+ assert seq_group.query_len is None # Decode
347
+
348
+ def _advance_step(self, model_input: StatefulModelInput,
349
+ out: SamplerOutput) -> StatefulModelInput:
350
+ frozen_model_input = model_input.frozen_model_input
351
+ assert frozen_model_input is not None
352
+ assert frozen_model_input.attn_metadata is not None
353
+
354
+ num_seqs = model_input.num_seqs
355
+ num_queries = model_input.num_queries
356
+ assert num_seqs > 0
357
+ assert num_queries > 0
358
+ assert num_seqs >= num_queries
359
+
360
+ attn_metadata = frozen_model_input.attn_metadata
361
+ assert isinstance(attn_metadata, FlashAttentionMetadata)
362
+ attn_metadata.advance_step(num_seqs, num_queries)
363
+
364
+ # Update GPU tensors
365
+ ops.advance_step(
366
+ num_seqs=num_seqs,
367
+ num_queries=num_queries,
368
+ block_size=self.block_size,
369
+ input_tokens=frozen_model_input.input_tokens,
370
+ sampled_token_ids=model_input.cached_outputs[-1].sampled_token_ids,
371
+ input_positions=frozen_model_input.input_positions,
372
+ seq_lens=attn_metadata.seq_lens_tensor,
373
+ slot_mapping=attn_metadata.slot_mapping,
374
+ block_tables=attn_metadata.block_tables)
375
+
376
+ if frozen_model_input.seq_lens is not None:
377
+ for i in range(num_queries):
378
+ frozen_model_input.seq_lens[i] = attn_metadata.seq_lens[i]
379
+
380
+ return model_input
381
+
382
+ def load_model(self) -> None:
383
+ return self._base_model_runner.load_model()
384
+
385
+ def save_sharded_state(
386
+ self,
387
+ path: str,
388
+ pattern: Optional[str] = None,
389
+ max_size: Optional[int] = None,
390
+ ) -> None:
391
+ return self._base_model_runner.save_sharded_state(
392
+ path, pattern, max_size)
393
+
394
+ def save_tensorized_model(self,
395
+ tensorizer_config: TensorizerConfig) -> None:
396
+ return self._base_model_runner.save_tensorized_model(tensorizer_config)
397
+
398
+ def profile_run(self) -> None:
399
+ return self._base_model_runner.profile_run()
400
+
401
+ def remove_all_loras(self):
402
+ return self._base_model_runner.remove_all_loras()
403
+
404
+ def capture_model(self, kv_caches: List[List]) -> None:
405
+ return self._base_model_runner.capture_model(kv_caches)
406
+
407
+ @property
408
+ def vocab_size(self) -> int:
409
+ return self._base_model_runner.vocab_size
410
+
411
+
412
+def _pythonize_sampler_output(model_input: StatefulModelInput,
413
+ output: SamplerOutput,
414
+ pinned_sampled_token_buffer: torch.Tensor,
415
+ sampled_token_ids: torch.Tensor) -> None:
416
+ """ This function is only called when the output tensors are ready.
417
+ See ModelOutput
418
+ """
419
+
420
+ assert model_input.frozen_model_input is not None
421
+
422
+ frozen_model_input = model_input.frozen_model_input
423
+ assert frozen_model_input.sampling_metadata is not None
424
+ # samples generation should have been skipped
425
+ assert not output.outputs
426
+
427
+ pinned_buffer = pinned_sampled_token_buffer[:model_input.num_queries]
428
+
429
+ # CPU GPU sync
430
+ pinned_buffer = pinned_buffer.copy_(sampled_token_ids, non_blocking=False)
431
+
432
+ # this will not block as the tensors are already on CPU
433
+ samples_list = pinned_buffer.tolist()
434
+
435
+ sampling_metadata = frozen_model_input.sampling_metadata
436
+
437
+ for (seq_group, sample_result) in zip(sampling_metadata.seq_groups,
438
+ samples_list):
439
+ seq_ids = seq_group.seq_ids
440
+ next_token_ids = sample_result
441
+ parent_ids = [0]
442
+ seq_outputs: List[SequenceOutput] = []
443
+ if seq_group.sampling_params.logits_processors:
444
+ assert len(seq_group.sampling_params.logits_processors) == 0, (
445
+ "Logits Processors are not supported in multi-step decoding")
446
+ for parent_id, next_token_id in zip(parent_ids, next_token_ids):
447
+ # TODO(will): support logprobs
448
+ # Hard coded logprob
449
+ seq_outputs.append(
450
+ SequenceOutput(seq_ids[parent_id], next_token_id,
451
+ {next_token_id: Logprob(logprob=-1)}))
452
+ output.outputs.append(CompletionSequenceGroupOutput(seq_outputs, None))
453
+ assert len(output.outputs) > 0
vllm/worker/multi_step_worker.py
new
+189
@@ -0,0 +1,189 @@
1
+from dataclasses import dataclass
2
+from typing import List, Optional, Tuple
3
+
4
+from vllm.distributed import broadcast_tensor_dict, get_pp_group
5
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
6
+from vllm.worker.model_runner_base import BroadcastableModelInput
7
+from vllm.worker.multi_step_model_runner import (MultiStepModelRunner,
8
+ StatefulModelInput)
9
+from vllm.worker.worker import Worker, WorkerInput
10
+
11
+
12
+@dataclass
13
+class MultiStepState:
14
+ worker_input: WorkerInput
15
+ model_input: StatefulModelInput
16
+
17
+
18
+class MultiStepWorker(Worker):
19
+
20
+ def __init__(self, *args, **kwargs):
21
+ super().__init__(*args, **kwargs)
22
+ base_model_runner = self.model_runner
23
+ # for multi-step model, wrap the model runner with MultiStepModelRunner
24
+ self.model_runner = MultiStepModelRunner(
25
+ base_model_runner,
26
+ base_model_runner.model_config,
27
+ base_model_runner.parallel_config,
28
+ base_model_runner.scheduler_config,
29
+ base_model_runner.device_config,
30
+ base_model_runner.cache_config,
31
+ load_config=base_model_runner.load_config,
32
+ lora_config=self.lora_config,
33
+ kv_cache_dtype=self.cache_config.cache_dtype,
34
+ is_driver_worker=base_model_runner.is_driver_worker,
35
+ prompt_adapter_config=base_model_runner.prompt_adapter_config,
36
+ observability_config=base_model_runner.observability_config,
37
+ )
38
+
39
+ pipeline_parallel_size = self.parallel_config.pipeline_parallel_size
40
+ self.multi_step_states: List[
41
+ Optional[MultiStepState]] = [None] * pipeline_parallel_size
42
+ self.temp_output = None
43
+
44
+ def _get_driver_input_and_broadcast(
45
+ self, execute_model_req: ExecuteModelRequest
46
+ ) -> Tuple[BroadcastableModelInput, WorkerInput]:
47
+ """
48
+ Get the driver input and broadcast it to other workers.
49
+ """
50
+ assert self.is_driver_worker
51
+ virtual_engine = execute_model_req.virtual_engine
52
+ is_first_multi_step = execute_model_req.is_first_multi_step
53
+ if is_first_multi_step:
54
+ # on first step we prepare the worker input and model input normally
55
+ worker_input: WorkerInput = self.prepare_worker_input(
56
+ execute_model_req=execute_model_req)
57
+ model_input: StatefulModelInput = (
58
+ self.model_runner.prepare_model_input(
59
+ execute_model_req.seq_group_metadata_list,
60
+ execute_model_req.virtual_engine,
61
+ execute_model_req.finished_requests_ids))
62
+ else:
63
+ # on subsequent steps we reuse the worker input and model input
64
+ multi_step_state = self.multi_step_states[virtual_engine]
65
+ worker_input = multi_step_state.worker_input
66
+ model_input = multi_step_state.model_input
67
+ frozen_model_input = model_input.frozen_model_input
68
+ assert frozen_model_input is not None
69
+ assert frozen_model_input.attn_metadata is not None
70
+ # clear the cached decode metadata so that it can be recomputed on
71
+ # the workers
72
+ frozen_model_input.attn_metadata._cached_decode_metadata = None
73
+
74
+ model_input.is_first_multi_step = is_first_multi_step
75
+ model_input.is_last_step = execute_model_req.is_last_step
76
+
77
+ if not is_first_multi_step:
78
+ # we broadcast the last sampled token ids to all TP workers so they
79
+ # can update their model input metadata in-place.
80
+ self._prepare_last_sampled_token_ids_for_tp_workers(
81
+ execute_model_req=execute_model_req, model_input=model_input)
82
+
83
+ if self.do_metadata_broadcast:
84
+ broadcast_data = worker_input.as_broadcastable_tensor_dict()
85
+ broadcast_data.update(model_input.as_broadcastable_tensor_dict())
86
+ broadcast_tensor_dict(broadcast_data, src=0)
87
+
88
+ return model_input, worker_input
89
+
90
+ def _prepare_last_sampled_token_ids_for_tp_workers(
91
+ self,
92
+ execute_model_req: ExecuteModelRequest,
93
+ model_input: StatefulModelInput,
94
+ ) -> None:
95
+ """
96
+ Prepare the last sampled token ids for TP workers. If it's the last
97
+ PP rank, then the last sampled token ids are already in the model_input.
98
+ If it is NOT the last PP rank, then we need to get the last sampled
99
+ token that is cached in the execute_model_req.
100
+ """
101
+ if get_pp_group().is_last_rank:
102
+ assert model_input.cached_outputs[
103
+ -1].sampler_output.sampled_token_ids is None
104
+ assert model_input.cached_outputs[-1].sampled_token_ids is not None
105
+ model_input.last_sampled_token_ids = model_input.cached_outputs[
106
+ -1].sampled_token_ids
107
+ # free sampled token ids from the previous step if it has been
108
+ # pythonized. Cannot free the last sampled token ids because
109
+ # we need it for GPU advance_step.
110
+ for output in model_input.cached_outputs[:-1]:
111
+ if output.pythonized:
112
+ output.sampled_token_ids = None
113
+ else:
114
+ # otherwise we need to get the cached sampled token ids from the
115
+ # execute_model_req
116
+ assert execute_model_req.last_sampled_token_ids is not None
117
+ model_input.last_sampled_token_ids = (
118
+ execute_model_req.last_sampled_token_ids.cuda())
119
+ model_input.add_sampler_output(
120
+ SamplerOutput(outputs=[], sampled_token_ids=None),
121
+ model_input.last_sampled_token_ids)
122
+
123
+ # free sampled token ids from the previous step.
124
+ # TODO(will) we could reuse the sampled token ids tensor from
125
+ # the previous step instead.
126
+ for output in model_input.cached_outputs[:-1]:
127
+ output.sampled_token_ids = None
128
+ assert model_input.cached_outputs[-1].sampled_token_ids is not None
129
+
130
+ def prepare_input(
131
+ self,
132
+ execute_model_req: Optional[ExecuteModelRequest] = None,
133
+ ) -> Optional[Tuple[StatefulModelInput, WorkerInput]]:
134
+ """
135
+ Depending on the current state of the request and multi step worker,
136
+ this method may skip the normal _prepare_model_input and
137
+ _prepare_worker_input methods and instead used cached values.
138
+ """
139
+ if self.is_driver_worker:
140
+ if execute_model_req is None:
141
+ if self.do_metadata_broadcast:
142
+ # This signals that there's no more requests to process for
143
+ # now. All workers are running infinite loop with
144
+ # broadcast_tensor_dict, and it stops the loop when the
145
+ # driver broadcasts an empty input. Send an empty input to
146
+ # notify all other workers to stop their execution loop.
147
+ broadcast_tensor_dict({}, src=0)
148
+ return None
149
+
150
+ virtual_engine = execute_model_req.virtual_engine
151
+ model_input, worker_input = self._get_driver_input_and_broadcast(
152
+ execute_model_req)
153
+ assert isinstance(model_input, StatefulModelInput)
154
+ if execute_model_req.is_first_multi_step:
155
+ # cache the worker input and model input for the next steps
156
+ self.multi_step_states[virtual_engine] = MultiStepState(
157
+ worker_input=worker_input, model_input=model_input)
158
+ # if TP workers
159
+ else:
160
+ broadcast_data = self._get_worker_input_from_broadcast()
161
+ # if the driver has sent an empty input, we should stop the worker
162
+ # loop
163
+ if broadcast_data is None:
164
+ return None
165
+ model_input, worker_input = broadcast_data
166
+ assert isinstance(model_input, StatefulModelInput)
167
+ virtual_engine = worker_input.virtual_engine
168
+ if model_input.is_first_multi_step:
169
+ pass
170
+ # TODO(will) Can cache the worker input and model input for the
171
+ # next steps. See below for details
172
+ else:
173
+ # TODO(will) possible to also cache and reuse the cached worker
174
+ # input and model input. The idea is essentially the delta
175
+ # optimization for model_inputs. Where the TP workers can cache
176
+ # the model input states and we only broadcast the delta need
177
+ # for the next step (sampled_token_ids from the previous step)
178
+
179
+ assert isinstance(model_input, StatefulModelInput)
180
+ # we need to update the last sampled token ids in the model
181
+ # input for the workers so that they can run inplace
182
+ # advance_step
183
+ model_input.add_sampler_output(
184
+ SamplerOutput(outputs=[], sampled_token_ids=None),
185
+ model_input.last_sampled_token_ids)
186
+
187
+ assert model_input is not None
188
+ assert worker_input is not None
189
+ return model_input, worker_input
vllm/worker/worker_base.py
+6
-4
@@ -16,7 +16,9 @@ from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
16
SamplerOutput)
17
from vllm.utils import (enable_trace_function_call_for_thread,
18
update_environment_variables)
19
-from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
19
+from vllm.worker.model_runner_base import (BroadcastableModelInput,
20
+ ModelRunnerBase,
21
+ ModelRunnerInputBase)
22
23
logger = init_logger(__name__)
24
@@ -220,7 +222,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
222
raise NotImplementedError
223
224
def _get_worker_input_from_broadcast(
223
- self) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
225
+ self) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
226
""" Get the worker input from the broadcasted tensor dict. """
227
assert self.do_metadata_broadcast
228
assert not self.is_driver_worker
@@ -237,7 +239,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
239
240
def _get_driver_input_and_broadcast(
241
self, execute_model_req: ExecuteModelRequest
240
- ) -> Tuple[ModelRunnerInputBase, WorkerInput]:
242
+ ) -> Tuple[BroadcastableModelInput, WorkerInput]:
243
""" Get the driver input and broadcast it to other workers. """
244
assert self.is_driver_worker
245
@@ -259,7 +261,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
261
def prepare_input(
262
self,
263
execute_model_req: Optional[ExecuteModelRequest] = None
262
- ) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
264
+ ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
265
"""
266
Prepare the inputs to ModelRunner and workers.
267
"""