@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 47b65a5508
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[core] Multi Step Scheduling (#7000)

Co-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com>

William Lin committed Aug 19, 2024 at 13:52 UTC 47b65a550866c7ffbd076ecb74106714838ce7da
13 files changed +1004 -34
.buildkite/test-pipeline.yaml
+9
@@ -311,6 +311,15 @@ steps:
311 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
312 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
313
314 +- label: Multi-step Tests (4 GPUs) # 10min
315 + working_dir: "/vllm-workspace/tests"
316 + num_gpus: 4
317 + source_file_dependencies:
318 + - vllm/
319 + - tests/multi_step/test_correctness.py
320 + commands:
321 + - pytest -v -s multi_step/test_correctness.py
322 +
323 - label: Pipeline Parallelism Test # 23min
324 working_dir: "/vllm-workspace/tests"
325 num_gpus: 4
tests/multi_step/__init__.py
tests/multi_step/test_correctness.py new
+85
@@ -0,0 +1,85 @@
1 +# Test the AsyncLLMEngine with multi-step-decoding
2 +
3 +from typing import List
4 +
5 +import pytest
6 +
7 +from ..utils import RemoteOpenAIServer
8 +
9 +MODELS = [
10 + "JackFram/llama-160m",
11 +]
12 +NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps
13 +NUM_PROMPTS = [10]
14 +
15 +DEFAULT_SERVER_ARGS: List[str] = [
16 + "--disable-log-requests",
17 + "--use-v2-block-manager",
18 + "--worker-use-ray",
19 + "--gpu-memory-utilization",
20 + "0.85",
21 + "--swap-space",
22 + "16",
23 +]
24 +
25 +
26 +async def completions_with_server_args(prompts: List[str], model_name: str,
27 + server_cli_args: List[str]):
28 +
29 + outputs = None
30 + with RemoteOpenAIServer(model_name, server_cli_args) as server:
31 + client = server.get_async_client()
32 + outputs = await client.completions.create(model=model_name,
33 + prompt=prompts,
34 + temperature=0,
35 + stream=False,
36 + max_tokens=5)
37 + assert outputs is not None
38 +
39 + return outputs
40 +
41 +
42 +@pytest.mark.parametrize("model", MODELS)
43 +@pytest.mark.parametrize(("tp_size, pp_size"), [
44 + (1, 1),
45 + (2, 2),
46 +])
47 +@pytest.mark.parametrize("eager_mode", [False, True])
48 +@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
49 +@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
50 +@pytest.mark.asyncio
51 +async def test_multi_step(example_prompts, model: str, tp_size: int,
52 + pp_size: int, eager_mode: int,
53 + num_scheduler_steps: int, num_prompts: int):
54 +
55 + prompts = example_prompts
56 + if len(prompts) < num_prompts:
57 + prompts = prompts * ((num_prompts // len(prompts)) + 1)
58 + prompts = prompts[:num_prompts]
59 + assert len(prompts) == num_prompts
60 +
61 + server_args = DEFAULT_SERVER_ARGS + ["--enforce-eager"]
62 + ms_server_args = DEFAULT_SERVER_ARGS + \
63 + ["--num-scheduler-steps", f"{num_scheduler_steps}"]
64 +
65 + if eager_mode:
66 + ms_server_args.append("--enforce-eager")
67 +
68 + distributed_args = [
69 + "--tensor-parallel-size",
70 + str(tp_size),
71 + "--pipeline-parallel-size",
72 + str(pp_size),
73 + ]
74 +
75 + ref_completions = await completions_with_server_args(
76 + prompts, model, server_args + distributed_args)
77 + test_completions = await completions_with_server_args(
78 + prompts, model, ms_server_args + distributed_args)
79 +
80 + def get_text_generations(completions):
81 + return [x.text for x in completions.choices]
82 +
83 + ref_generations = get_text_generations(ref_completions)
84 + test_generations = get_text_generations(test_completions)
85 + assert ref_generations == test_generations
tests/worker/test_model_input.py
+77
@@ -10,6 +10,7 @@ from vllm.model_executor.pooling_metadata import PoolingMetadata
10 from vllm.worker.embedding_model_runner import (
11 ModelInputForGPUWithPoolingMetadata)
12 from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
13 +from vllm.worker.multi_step_model_runner import StatefulModelInput
14
15
16 class MockAttentionBackend(AttentionBackend):
@@ -154,3 +155,79 @@ def test_embedding_model_runner_input():
155 None) == getattr(attn_metadata, field.name, None)
156 # Pooling metadata is not broadcast.
157 assert received_model_input.pooling_metadata is None
158 +
159 +
160 +def test_multi_step_model_runner_input():
161 + sampling_metadata = SamplingMetadata(
162 + ["seq_group"],
163 + "selected_token_indices",
164 + "categorized_sample_indices",
165 + "num_prompts",
166 + )
167 + attn_metadata = AttentionMetadata(
168 + num_prefills=1,
169 + num_prefill_tokens=2,
170 + num_decode_tokens=3,
171 + slot_mapping=torch.zeros(1),
172 + )
173 + frozen_model_input = ModelInputForGPUWithSamplingMetadata(
174 + input_tokens=torch.ones(10),
175 + input_positions=torch.ones(10),
176 + sampling_metadata=sampling_metadata,
177 + attn_metadata=attn_metadata)
178 +
179 + model_input = StatefulModelInput(
180 + frozen_model_input=frozen_model_input,
181 + is_last_step=True,
182 + is_first_multi_step=False,
183 + current_step=4,
184 + last_sampled_token_ids=torch.ones((10, 1)),
185 + is_multi_step=True,
186 + num_queries=8,
187 + num_seqs=5,
188 + cached_outputs=[],
189 + )
190 +
191 + assert isinstance(model_input, StatefulModelInput)
192 +
193 + # Test round trip serialization.
194 + tensor_dict = model_input.as_broadcastable_tensor_dict()
195 + attn_backend = MockAttentionBackend()
196 + received_model_input = (StatefulModelInput.from_broadcasted_tensor_dict(
197 + tensor_dict, attn_backend=attn_backend))
198 +
199 + receieved_frozen_input = received_model_input.frozen_model_input
200 +
201 + # Check that received copy has correct values.
202 + assert isinstance(received_model_input, StatefulModelInput)
203 + assert receieved_frozen_input.input_tokens is not None
204 + assert (receieved_frozen_input.input_tokens ==
205 + frozen_model_input.input_tokens).all()
206 + assert receieved_frozen_input.input_positions is not None
207 + assert (receieved_frozen_input.input_positions ==
208 + frozen_model_input.input_positions).all()
209 + assert receieved_frozen_input.multi_modal_kwargs is None
210 + assert (frozen_model_input.multi_modal_kwargs ==
211 + frozen_model_input.multi_modal_kwargs)
212 + assert receieved_frozen_input.lora_requests is None
213 + assert (receieved_frozen_input.lora_requests ==
214 + frozen_model_input.lora_requests)
215 + assert receieved_frozen_input.lora_mapping is None
216 + assert (
217 + receieved_frozen_input.lora_mapping == frozen_model_input.lora_mapping)
218 + for field in dataclasses.fields(AttentionMetadata):
219 + assert getattr(receieved_frozen_input.attn_metadata, field.name,
220 + None) == getattr(attn_metadata, field.name, None)
221 + # For sampling metadata, only selected_token_indices is copied.
222 + assert (receieved_frozen_input.sampling_metadata.selected_token_indices ==
223 + sampling_metadata.selected_token_indices)
224 + assert receieved_frozen_input.sampling_metadata.seq_groups is None
225 +
226 + # check non frozen fields
227 + assert received_model_input.is_last_step == model_input.is_last_step
228 + assert (received_model_input.is_first_multi_step ==
229 + model_input.is_first_multi_step)
230 + assert received_model_input.current_step == model_input.current_step
231 + assert (received_model_input.last_sampled_token_ids ==
232 + model_input.last_sampled_token_ids).all()
233 + assert received_model_input.is_multi_step == model_input.is_multi_step
vllm/engine/arg_utils.py
+6 -1
@@ -853,6 +853,12 @@ class EngineArgs:
853 "in low performance due to small KV cache space. Consider "
854 "setting --max-model-len to a smaller value.", max_model_len)
855
856 + if self.num_scheduler_steps > 1 and not self.use_v2_block_manager:
857 + self.use_v2_block_manager = True
858 + logger.warning(
859 + "Enabled BlockSpaceManagerV2 because it is "
860 + "required for multi-step (--num-scheduler-steps > 1)")
861 +
862 speculative_config = SpeculativeConfig.maybe_create_spec_config(
863 target_model_config=model_config,
864 target_parallel_config=parallel_config,
@@ -881,7 +887,6 @@ class EngineArgs:
887 )
888
889 if self.num_scheduler_steps > 1:
884 - raise NotImplementedError("Multi-step is not yet supported.")
890 if speculative_config is not None:
891 raise ValueError("Speculative decoding is not supported with "
892 "multi-step (--num-scheduler-steps > 1)")
vllm/engine/async_llm_engine.py
+127 -8
@@ -1,9 +1,11 @@
1 import asyncio
2 import time
3 +from dataclasses import dataclass
4 from functools import partial
5 from typing import (AsyncGenerator, Callable, Dict, Iterable, List, Mapping,
6 Optional, Set, Tuple, Type, Union)
7
8 +import torch
9 from transformers import PreTrainedTokenizer
10 from typing_extensions import assert_never
11
@@ -27,7 +29,8 @@ from vllm.outputs import EmbeddingRequestOutput, RequestOutput
29 from vllm.pooling_params import PoolingParams
30 from vllm.prompt_adapter.request import PromptAdapterRequest
31 from vllm.sampling_params import SamplingParams
30 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
32 +from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
33 + SequenceGroupMetadata)
34 from vllm.usage.usage_lib import UsageContext
35 from vllm.utils import print_warning_once
36
@@ -249,9 +252,25 @@ class RequestTracker:
252 return not self._new_requests.empty()
253
254
255 +@dataclass
256 +class SchedulerOutputState:
257 + """Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
258 + last_output: Optional[SamplerOutput] = None
259 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
260 + scheduler_outputs: Optional[SchedulerOutputs] = None
261 +
262 +
263 class _AsyncLLMEngine(LLMEngine):
264 """Extension of LLMEngine to add async methods."""
265
266 + def __init__(self, *args, **kwargs):
267 + super().__init__(*args, **kwargs)
268 + pipeline_parallel_size = \
269 + self.parallel_config.pipeline_parallel_size
270 + self.cached_scheduler_outputs = [
271 + SchedulerOutputState() for _ in range(pipeline_parallel_size)
272 + ]
273 +
274 async def step_async(
275 self, virtual_engine: int
276 ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
@@ -264,13 +283,39 @@ class _AsyncLLMEngine(LLMEngine):
283 and updates the scheduler with the model outputs. Finally, it decodes
284 the sequences and returns the newly generated results.
285 """
267 - seq_group_metadata_list, scheduler_outputs = self.scheduler[
268 - virtual_engine].schedule()
286 + # these are cached outputs from previous iterations. None if on first
287 + # iteration
288 + cached_outputs = self.cached_scheduler_outputs[virtual_engine]
289 + seq_group_metadata_list = cached_outputs.seq_group_metadata_list
290 + scheduler_outputs = cached_outputs.scheduler_outputs
291 + # skip the scheduler if there are any remaining steps in the seq groups.
292 + # This ensures that the scheduler is only called again when the current
293 + # batch has completed.
294 + if not self._has_remaining_steps(seq_group_metadata_list):
295 + seq_group_metadata_list, scheduler_outputs = self.scheduler[
296 + virtual_engine].schedule()
297 +
298 + if (self.scheduler_config.is_multi_step
299 + and scheduler_outputs.num_lookahead_slots > 0):
300 + # cache the scheduler outputs for the next iteration if we have
301 + # lookahead slots
302 + self._cache_scheduler_outputs_for_multi_step(
303 + virtual_engine, seq_group_metadata_list, scheduler_outputs)
304 +
305 + assert seq_group_metadata_list is not None
306 + assert scheduler_outputs is not None
307
308 if not scheduler_outputs.is_empty():
271 - # Execute the model.
309 finished_requests_ids = self.scheduler[
310 virtual_engine].get_and_reset_finished_requests_ids()
311 +
312 + # Check if we have a cached last_output from the previous iteration.
313 + # For supporting PP this is probably the best way to pass the
314 + # sampled_token_ids, as a separate broadcast over all the PP stages
315 + # will cause one virtual engine's microbatch to block the pipeline.
316 + last_sampled_token_ids = \
317 + self._get_last_sampled_token_ids(virtual_engine)
318 +
319 execute_model_req = ExecuteModelRequest(
320 seq_group_metadata_list=seq_group_metadata_list,
321 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
@@ -279,15 +324,35 @@ class _AsyncLLMEngine(LLMEngine):
324 virtual_engine=virtual_engine,
325 num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
326 running_queue_size=scheduler_outputs.running_queue_size,
282 - finished_requests_ids=finished_requests_ids)
327 + finished_requests_ids=finished_requests_ids,
328 + # We use ExecuteModelRequest to pass the last sampled_token_ids
329 + # to each of the non-last PP stages for in-place prepare_input.
330 + last_sampled_token_ids=last_sampled_token_ids)
331 + # Execute the model.
332 output = await self.model_executor.execute_model_async(
333 execute_model_req)
334 + # we need to do this here so that last step's sampled_token_ids can
335 + # be passed to the next iteration for PP.
336 + if self.scheduler_config.is_multi_step:
337 + self._update_cached_scheduler_output(virtual_engine, output)
338 else:
339 output = []
340
288 - request_outputs = self._process_model_outputs(
289 - output, scheduler_outputs.scheduled_seq_groups,
290 - scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
341 + # Finish the current step for all the sequence groups.
342 + if self.scheduler_config.is_multi_step:
343 + for seq_group in seq_group_metadata_list:
344 + seq_group.finish_step()
345 +
346 + if not self._has_remaining_steps(seq_group_metadata_list):
347 + # clear the cache if we have finished all the steps
348 + if self.scheduler_config.is_multi_step:
349 + self.cached_scheduler_outputs[
350 + virtual_engine] = SchedulerOutputState()
351 + request_outputs = self._process_model_outputs(
352 + output, scheduler_outputs.scheduled_seq_groups,
353 + scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
354 + else:
355 + request_outputs = []
356
357 # Log stats.
358 self.do_log_stats(scheduler_outputs, output)
@@ -297,6 +362,60 @@ class _AsyncLLMEngine(LLMEngine):
362
363 return request_outputs
364
365 + def _has_remaining_steps(
366 + self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
367 + ) -> bool:
368 + if (not self.scheduler_config.is_multi_step
369 + or not seq_group_metadata_list):
370 + return False
371 +
372 + # TODO(will) this is a sanity check for nowto make sure that all the
373 + # seqs are on the same steps. Eventually we will want to do some sort of
374 + # dynamic scheduling when doing multi-step decoding.
375 + ref_remaining_steps = seq_group_metadata_list[0].state.remaining_steps
376 + if any([
377 + seq_group.state.remaining_steps != ref_remaining_steps
378 + for seq_group in seq_group_metadata_list[1:]
379 + ]):
380 + raise AssertionError(("All running sequence groups should "
381 + "have the same remaining steps."))
382 +
383 + return ref_remaining_steps > 0
384 +
385 + def _cache_scheduler_outputs_for_multi_step(
386 + self, virtual_engine: int,
387 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
388 + scheduler_outputs: SchedulerOutputs) -> None:
389 + self.cached_scheduler_outputs[
390 + virtual_engine].seq_group_metadata_list = seq_group_metadata_list
391 + self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
392 + scheduler_outputs
393 + self.cached_scheduler_outputs[virtual_engine].last_output = None
394 +
395 + def _get_last_sampled_token_ids(
396 + self, virtual_engine: int) -> Optional[torch.Tensor]:
397 + cached_last_output = self.cached_scheduler_outputs[
398 + virtual_engine].last_output
399 + if (self.scheduler_config.is_multi_step
400 + and self.parallel_config.pipeline_parallel_size > 1
401 + and cached_last_output is not None
402 + and cached_last_output.sampled_token_ids_cpu is not None):
403 + return cached_last_output.sampled_token_ids_cpu
404 + return None
405 +
406 + def _update_cached_scheduler_output(
407 + self, virtual_engine: int,
408 + output: List[Optional[SamplerOutput]]) -> None:
409 + if (self.parallel_config.pipeline_parallel_size > 1 and len(output) > 0
410 + and output[0] is not None):
411 + last_output = output[-1]
412 + assert last_output is not None
413 + assert last_output.sampled_token_ids_cpu is not None
414 + assert last_output.sampled_token_ids is None
415 + assert last_output.sampled_token_probs is None
416 + self.cached_scheduler_outputs[
417 + virtual_engine].last_output = last_output
418 +
419 async def stop_remote_worker_execution_loop_async(self) -> None:
420 """Stop the remote worker execution loop."""
421 await self.model_executor.stop_remote_worker_execution_loop_async()
vllm/executor/gpu_executor.py
+10 -4
@@ -69,13 +69,19 @@ class GPUExecutor(ExecutorBase):
69 distributed_init_method: Optional[str] = None) -> Dict:
70 worker_kwargs = self._get_worker_kwargs(local_rank, rank,
71 distributed_init_method)
72 - if self.speculative_config is None:
73 - worker_kwargs.update(worker_module_name="vllm.worker.worker",
74 - worker_class_name="Worker")
75 - else:
72 +
73 + if self.scheduler_config.is_multi_step:
74 + worker_kwargs.update(
75 + worker_module_name="vllm.worker.multi_step_worker",
76 + worker_class_name="MultiStepWorker")
77 + elif self.speculative_config:
78 worker_kwargs.update(
79 worker_module_name="vllm.spec_decode.spec_decode_worker",
80 worker_class_name="create_spec_worker")
81 + else:
82 + worker_kwargs.update(worker_module_name="vllm.worker.worker",
83 + worker_class_name="Worker")
84 +
85 return worker_kwargs
86
87 def _create_worker(self,
vllm/executor/ray_gpu_executor.py
+3
@@ -94,6 +94,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
94 if self.speculative_config is not None:
95 worker_module_name = "vllm.spec_decode.spec_decode_worker"
96 worker_class_name = "create_spec_worker"
97 + elif self.scheduler_config.is_multi_step:
98 + worker_module_name = "vllm.worker.multi_step_worker"
99 + worker_class_name = "MultiStepWorker"
100 else:
101 worker_module_name = "vllm.worker.worker"
102 worker_class_name = "Worker"
vllm/sequence.py
+5 -5
@@ -9,7 +9,6 @@ from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
9 Tuple, Union, cast)
10
11 import msgspec
12 -import numpy
12 import torch
13
14 from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
@@ -1082,7 +1081,10 @@ class SamplerOutput(
1081
1082 # On-device tensor containing the sampled token ids.
1083 sampled_token_ids: Optional[torch.Tensor] = None
1085 - sampled_token_ids_numpy: Optional[numpy.ndarray] = None
1084 + # CPU tensor containing the sampled token ids. Used during multi-step to
1085 + # return the sampled token ids from last rank to AsyncLLMEngine to be
1086 + # 'broadcasted' to all other PP ranks for next step.
1087 + sampled_token_ids_cpu: Optional[torch.Tensor] = None
1088
1089 # Spec decode metrics populated by workers.
1090 spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
@@ -1257,9 +1259,7 @@ class ExecuteModelRequest(
1259 assert len(self.seq_group_metadata_list) > 0
1260 first_seq_group = self.seq_group_metadata_list[0]
1261 assert first_seq_group.state is not None
1260 - num_steps = first_seq_group.state.num_steps
1261 - current_step = first_seq_group.state.current_step
1262 - return num_steps - current_step == 1
1262 + return first_seq_group.state.remaining_steps == 1
1263
1264 @property
1265 def current_step(self) -> int:
vllm/worker/model_runner_base.py
+34 -12
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
14 from vllm.attention.backends.abstract import AttentionBackend
15 from vllm.model_executor import SamplingMetadata
16
17 -T = TypeVar('T', bound="ModelRunnerInputBase")
17 +T = TypeVar('T', bound="BroadcastableModelInput")
18
19
20 def _add_attn_metadata_broadcastable_dict(
@@ -81,18 +81,26 @@ def _add_sampling_metadata_broadcastable_dict(
81 sampling_metadata.selected_token_indices)
82
83
84 -@dataclasses.dataclass(frozen=True)
85 -class ModelRunnerInputBase(ABC):
86 - """Local inputs to each worker's model runner. May contain
87 - device-specific data. Different worker backends may have different methods
88 - of converting from the global ExecuteModelRequest produced by the LLM
89 - engine to the worker-local ModelRunnerInputBase objects.
90 -
91 - Model runners that support multi-GPU execution should define a
92 - ModelRunnerInputBase subclass, add their required fields, and specify how to
93 - serialize/deserialize a ModelInput for broadcast between workers.
84 +def _init_frozen_model_input_from_tensor_dict(
85 + frozen_model_input_cls: Type["ModelRunnerInputBase"],
86 + tensor_dict: Dict[str, Any]) -> Dict[str, Any]:
87 """
88 + Helper method to initialize a frozen ModelInput based on broadcastable
89 + """
90 + valid_tensor_kwargs = {}
91 + for field in dataclasses.fields(frozen_model_input_cls):
92 + val = tensor_dict.pop(field.name, None)
93 + if val is not None:
94 + valid_tensor_kwargs[field.name] = val
95 +
96 + frozen_model_input = frozen_model_input_cls(**valid_tensor_kwargs)
97 + tensor_dict["frozen_model_input"] = frozen_model_input
98 + return tensor_dict
99
100 +
101 +class BroadcastableModelInput(ABC):
102 +
103 + @abstractmethod
104 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
105 """
106 Extract broadcastable fields. Override for fields that require some
@@ -109,11 +117,25 @@ class ModelRunnerInputBase(ABC):
117 ) -> T:
118 """
119 Pop fields from the given tensor_dict and populate a new instance of
112 - ModelRunnerInputBase.
120 + BroadcastableModelInput.
121 """
122 raise NotImplementedError
123
124
125 +@dataclasses.dataclass(frozen=True)
126 +class ModelRunnerInputBase(BroadcastableModelInput):
127 + """Local inputs to each worker's model runner. May contain
128 + device-specific data. Different worker backends may have different methods
129 + of converting from the global ExecuteModelRequest produced by the LLM
130 + engine to the worker-local ModelRunnerInputBase objects.
131 +
132 + Model runners that support multi-GPU execution should define a
133 + ModelRunnerInputBase subclass, add their required fields, and specify how to
134 + serialize/deserialize a ModelInput for broadcast between workers.
135 + """
136 + pass
137 +
138 +
139 class ModelRunnerInputBuilderBase(ABC, Generic[T]):
140 """A builder to create ModelRunnerInputBase objects.
141 """
vllm/worker/multi_step_model_runner.py new
+453
@@ -0,0 +1,453 @@
1 +from dataclasses import dataclass, field
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
3 +
4 +try:
5 + from vllm.attention.backends.flash_attn import FlashAttentionMetadata
6 +except ModuleNotFoundError:
7 + # vllm_flash_attn is not installed, use the identical ROCm FA metadata
8 + from vllm.attention.backends.rocm_flash_attn import (
9 + ROCmFlashAttentionMetadata as FlashAttentionMetadata)
10 +
11 +import torch
12 +
13 +from vllm import _custom_ops as ops
14 +from vllm.distributed import get_pp_group
15 +from vllm.logger import init_logger
16 +from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
17 + Logprob, SamplerOutput, SequenceGroupMetadata,
18 + SequenceOutput)
19 +from vllm.worker.model_runner import (GPUModelRunnerBase,
20 + ModelInputForGPUWithSamplingMetadata)
21 +from vllm.worker.model_runner_base import (
22 + BroadcastableModelInput, _init_attn_metadata_from_tensor_dict,
23 + _init_frozen_model_input_from_tensor_dict,
24 + _init_sampling_metadata_from_tensor_dict)
25 +
26 +from ..model_executor.model_loader.tensorizer import TensorizerConfig
27 +
28 +if TYPE_CHECKING:
29 + from vllm.attention.backends.abstract import AttentionBackend
30 +
31 +logger = init_logger(__name__)
32 +
33 +
34 +@dataclass
35 +class ModelOutput:
36 + """The output of a single model forward pass.
37 +
38 + The sampler_output_ready_event is set when the tensors in
39 + sampler_output are ready (the model+sampler forward pass has
40 + completed). We use the event to synchronize the GPU->CPU transfer,
41 + which we want to only run when the data has been written to the
42 + GPU tensors. Until the event is ready, the tensors in sampler_output
43 + will have garbage data.
44 +
45 + There are two scenarios:
46 + 1. The output tensors are ready and we can pythonize them immediately.
47 + 2. The output tensors are not ready and we need to wait for the event to be
48 + ready.
49 + """
50 + sampler_output: SamplerOutput
51 + sampler_output_ready_event: torch.cuda.Event
52 + sampled_token_ids: Optional[torch.Tensor] = None
53 + pythonized: bool = False
54 +
55 + def pythonize(self, input_metadata: "StatefulModelInput",
56 + copy_stream: torch.cuda.Stream,
57 + pinned_sampled_token_buffer: torch.Tensor) -> None:
58 + """Pythonize the output. Blocking."""
59 + if not self.pythonized:
60 + self._pythonize_sampler_output(input_metadata, copy_stream,
61 + pinned_sampled_token_buffer, True)
62 + self.pythonized = True
63 +
64 + def maybe_pythonize(self, input_metadata: "StatefulModelInput",
65 + copy_stream: torch.cuda.Stream,
66 + pinned_sampled_token_buffer: torch.Tensor) -> None:
67 + """Pythonize the output if ready, else return None. Non-blocking."""
68 + if not self.pythonized:
69 + self.pythonized = self._pythonize_sampler_output(
70 + input_metadata, copy_stream, pinned_sampled_token_buffer,
71 + False)
72 +
73 + def _pythonize_sampler_output(self, input_metadata: "StatefulModelInput",
74 + copy_stream: torch.cuda.Stream,
75 + pinned_sampled_token_buffer: torch.Tensor,
76 + blocking: bool) -> bool:
77 + """
78 + If blocking is set, will block until the forward pass for the output is
79 + ready and pythonize the output.
80 + """
81 + assert self.sampled_token_ids is not None
82 + if not blocking and not self.sampler_output_ready_event.query():
83 + return False
84 +
85 + if blocking:
86 + self.sampler_output_ready_event.synchronize()
87 + with torch.cuda.stream(copy_stream):
88 + _pythonize_sampler_output(input_metadata, self.sampler_output,
89 + pinned_sampled_token_buffer,
90 + self.sampled_token_ids)
91 + return True
92 +
93 +
94 +@dataclass(frozen=False)
95 +class StatefulModelInput(BroadcastableModelInput):
96 + # actual frozen model input dataclass passed to _base_model_runner
97 + frozen_model_input: Optional[ModelInputForGPUWithSamplingMetadata] = None
98 +
99 + # list of model outputs for each step, may not be all pythonized
100 + cached_outputs: List[ModelOutput] = field(default_factory=list)
101 +
102 + # used to pass sampled token ids from the last step to the current step for
103 + # TP workers. Used to append to end of outputs and used by advance_step
104 + last_sampled_token_ids: Optional[torch.Tensor] = None
105 + current_step: int = 0
106 + is_multi_step: bool = True
107 + is_last_step: bool = False
108 + is_first_multi_step: bool = False
109 + # ping-pong data structures for multi-step to wait on the previous step
110 + step_cuda_events: List[torch.cuda.Event] = field(
111 + default_factory=lambda: [torch.cuda.Event(blocking=True)] * 2)
112 + num_seqs: int = -1
113 + num_queries: int = -1
114 +
115 + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
116 + assert self.frozen_model_input is not None
117 + tensor_dict = self.frozen_model_input.as_broadcastable_tensor_dict()
118 + new_tensor_dict = {
119 + 'last_sampled_token_ids': self.last_sampled_token_ids,
120 + 'current_step': self.current_step,
121 + 'is_multi_step': self.is_multi_step,
122 + 'is_last_step': self.is_last_step,
123 + 'is_first_multi_step': self.is_first_multi_step,
124 + 'num_seqs': self.num_seqs,
125 + 'num_queries': self.num_queries,
126 + }
127 + tensor_dict.update(new_tensor_dict)
128 + return tensor_dict
129 +
130 + @classmethod
131 + def from_broadcasted_tensor_dict(
132 + cls,
133 + tensor_dict: Dict[str, Any],
134 + attn_backend: Optional["AttentionBackend"] = None,
135 + ) -> "StatefulModelInput":
136 + tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
137 + if attn_backend is not None:
138 + tensor_dict = _init_attn_metadata_from_tensor_dict(
139 + attn_backend, tensor_dict)
140 + tensor_dict = _init_frozen_model_input_from_tensor_dict(
141 + ModelInputForGPUWithSamplingMetadata, tensor_dict)
142 +
143 + return cls(**tensor_dict)
144 +
145 + def record_step_event(self, current_stream: torch.cuda.Stream):
146 + # record the event for the current step so that the next step can sync
147 + # on it. We modulo by 2 to keep the events in a circular buffer and
148 + # support any attn backends that may be supported in the future. ie
149 + # Flashinfer would want two DecodeWrappers to overlap the CPU and GPU.
150 + self.step_cuda_events[self.current_step & 1] = \
151 + torch.cuda.Event(blocking=True)
152 + self.step_cuda_events[self.current_step & 1].record(current_stream)
153 +
154 + def wait_previous_step(self):
155 + # These cuda events are an explicit synchronization to ensure that
156 + # advance_step() (for other attn backends that may be supported in the
157 + # future) do not clobber any data structures that is also used by any
158 + # enqueued forwards steps. For distributed case, only a single event is
159 + # needed, but for single GPU case, since we can let the CPU run much
160 + # further ahead, two events allow us to overlap the advance_step with
161 + # the previous forward (ie using two DecodeWrappers for flashinfer
162 + # backend)
163 + self.step_cuda_events[(self.current_step + 1) & 1].wait()
164 +
165 + def add_sampler_output(self,
166 + sampler_output: SamplerOutput,
167 + sampled_token_ids: Optional[torch.Tensor] = None):
168 + self.cached_outputs.append(
169 + ModelOutput(sampler_output=sampler_output,
170 + sampler_output_ready_event=None,
171 + sampled_token_ids=sampled_token_ids,
172 + pythonized=False))
173 +
174 +
175 +# MutableModelInputForGPUWithMultiStepMetadata is not subclass of
176 +# ModelInputForGPU but it wraps the actual input dataclass and adds multi-step
177 +# metadata
178 +# mypy: disable-error-code=type-var
179 +class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
180 + # mypy: enable-error-code=type-var
181 +
182 + def __init__(self, base_model_runner: GPUModelRunnerBase, *args, **kwargs):
183 + super().__init__(*args, **kwargs)
184 +
185 + # uses the base model runner to execute the model and wraps it with
186 + # multi-step logic
187 + self._base_model_runner: GPUModelRunnerBase = base_model_runner
188 +
189 + self.is_multi_step = self.scheduler_config.is_multi_step
190 + # used to copy tensors from GPU to CPU asynchronously
191 + self._copy_stream = torch.cuda.Stream()
192 + self.pinned_sampled_token_ids: Optional[torch.Tensor] = None
193 +
194 + def make_model_input_from_broadcasted_tensor_dict(
195 + self, tensor_dict: Dict[str, Any]) -> StatefulModelInput:
196 + model_input = (StatefulModelInput.from_broadcasted_tensor_dict(
197 + tensor_dict,
198 + attn_backend=self.attn_backend,
199 + ))
200 + return model_input
201 +
202 + def prepare_model_input(
203 + self,
204 + seq_group_metadata_list: List[SequenceGroupMetadata],
205 + virtual_engine: int = 0,
206 + finished_requests_ids: Optional[List[str]] = None
207 + ) -> StatefulModelInput:
208 + frozen_model_input = self._base_model_runner.prepare_model_input(
209 + seq_group_metadata_list, virtual_engine, finished_requests_ids)
210 +
211 + model_input = StatefulModelInput(
212 + frozen_model_input=frozen_model_input,
213 + num_seqs=len(frozen_model_input.seq_lens),
214 + num_queries=len(frozen_model_input.query_lens),
215 + )
216 + return model_input
217 +
218 + @torch.inference_mode()
219 + def execute_model(
220 + self,
221 + model_input: StatefulModelInput,
222 + kv_caches: List[torch.Tensor],
223 + intermediate_tensors: Optional[IntermediateTensors] = None,
224 + num_steps: int = 1,
225 + ) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]:
226 + """
227 + Execute the model for a single step and update multi-step
228 + metadata
229 + """
230 + assert num_steps == 1, "MultiStepModelRunner only supports num_steps=1"
231 + frozen_model_input = model_input.frozen_model_input
232 + assert frozen_model_input is not None
233 +
234 + # path for warm up runs
235 + if not model_input.is_multi_step:
236 + return self._base_model_runner.execute_model(
237 + frozen_model_input, kv_caches, intermediate_tensors, num_steps)
238 +
239 + # make sure we skip the sampler on the lask rank and only pythonize
240 + # if CPU is ahead.
241 + if self.is_driver_worker and get_pp_group().is_last_rank:
242 + if self.pinned_sampled_token_ids is None:
243 + self.pinned_sampled_token_ids = torch.zeros(
244 + (self.scheduler_config.max_num_seqs, 1),
245 + dtype=torch.long,
246 + device="cpu",
247 + pin_memory=True)
248 +
249 + self._base_model_runner.model.sampler.include_gpu_probs_tensor = (
250 + True)
251 + if frozen_model_input.sampling_metadata:
252 + frozen_model_input.sampling_metadata.skip_sampler_cpu_output = (
253 + True)
254 +
255 + # some pre-execute model logic for multi-step:
256 + # - if it's the first step, we need to reset the sampling tensors
257 + # - if it's not the first step, we need to advance the step using the
258 + # appended sampler output from last iteration
259 + # - also maybe pythonize if CPU is ahead of GPU
260 +
261 + current_stream = torch.cuda.current_stream()
262 + if not model_input.is_first_multi_step:
263 + # Explicitly block on the previous step's forward to make sure we
264 + # don't clobber any GPU tensors still in use.
265 + # This is not needed for flashattn backend, but for other attn
266 + # backends such as flashinfer that performs extra CPU operations on
267 + # input metadata we may need to synchronize any CPU operations that
268 + # might clobber enqueued forwards. (prevents CPU from running too
269 + # far ahead if needed)
270 + model_input.wait_previous_step()
271 + model_input = self._advance_step(
272 + model_input, model_input.cached_outputs[-1].sampler_output)
273 +
274 + # Execute the model
275 + output = self._base_model_runner.execute_model(frozen_model_input,
276 + kv_caches,
277 + intermediate_tensors,
278 + num_steps=1)
279 +
280 + # record the event for the current step so that the next step can sync
281 + model_input.record_step_event(current_stream)
282 +
283 + if get_pp_group().is_last_rank and self.is_driver_worker:
284 + assert len(
285 + output
286 + ) == 1, "MultiStepModelRunner requires single-step base_models"
287 +
288 + # event for the pythonization so that we only pythonize if the
289 + # tensors are ready. May be able to be combined with the step event
290 + output_ready_event = torch.cuda.Event()
291 + output_ready_event.record(current_stream)
292 + if self.parallel_config.pipeline_parallel_size > 1:
293 + output[0].sampled_token_ids_cpu = output[
294 + 0].sampled_token_ids.cpu()
295 + model_input.cached_outputs.append(
296 + ModelOutput(output[0], output_ready_event,
297 + output[0].sampled_token_ids, False))
298 + # make sure we dont try to serialize any GPU tensors
299 + output[0].sampled_token_ids = None
300 + output[0].sampled_token_probs = None
301 + output[0].logprobs = None
302 + # Pythonize the output if CPU is ahead and the previous step is
303 + # ready.
304 + for model_output in model_input.cached_outputs:
305 + model_output.maybe_pythonize(model_input, self._copy_stream,
306 + self.pinned_sampled_token_ids)
307 +
308 + model_input.current_step += 1
309 +
310 + if not get_pp_group().is_last_rank:
311 + # Should be IntermediateTensors
312 + assert isinstance(output, IntermediateTensors)
313 + return output
314 + if not self.is_driver_worker:
315 + return []
316 +
317 + # Pythonize the output and block if needed since it is the last step
318 + if model_input.is_last_step:
319 + outputs = []
320 + for output in model_input.cached_outputs:
321 + output.pythonize(model_input, self._copy_stream,
322 + self.pinned_sampled_token_ids)
323 + outputs.append(output.sampler_output)
324 + return outputs
325 +
326 + # should be [SamplerOutput]
327 + return output
328 +
329 + def _update_sampling_metadata(self, sampling_metadata, num_seqs,
330 + num_queries):
331 +
332 + assert sampling_metadata.num_prompts == 0
333 + assert len(sampling_metadata.seq_groups) == num_queries
334 + assert sampling_metadata.selected_token_indices.shape == (
335 + num_queries, )
336 + # assert sampling_metadata.categorized_sample_indices == TODO: Add if needed # noqa: E501
337 +
338 + # Verify that all sequences are decodes
339 + for i in range(num_queries):
340 + seq_group = sampling_metadata.seq_groups[i]
341 +
342 + assert seq_group.is_prompt is False # No prompt
343 + assert seq_group.prompt_logprob_indices == [] # No prompt
344 + assert seq_group.sample_indices == [i] # Simple
345 + assert seq_group.seq_len is None # Decode
346 + assert seq_group.query_len is None # Decode
347 +
348 + def _advance_step(self, model_input: StatefulModelInput,
349 + out: SamplerOutput) -> StatefulModelInput:
350 + frozen_model_input = model_input.frozen_model_input
351 + assert frozen_model_input is not None
352 + assert frozen_model_input.attn_metadata is not None
353 +
354 + num_seqs = model_input.num_seqs
355 + num_queries = model_input.num_queries
356 + assert num_seqs > 0
357 + assert num_queries > 0
358 + assert num_seqs >= num_queries
359 +
360 + attn_metadata = frozen_model_input.attn_metadata
361 + assert isinstance(attn_metadata, FlashAttentionMetadata)
362 + attn_metadata.advance_step(num_seqs, num_queries)
363 +
364 + # Update GPU tensors
365 + ops.advance_step(
366 + num_seqs=num_seqs,
367 + num_queries=num_queries,
368 + block_size=self.block_size,
369 + input_tokens=frozen_model_input.input_tokens,
370 + sampled_token_ids=model_input.cached_outputs[-1].sampled_token_ids,
371 + input_positions=frozen_model_input.input_positions,
372 + seq_lens=attn_metadata.seq_lens_tensor,
373 + slot_mapping=attn_metadata.slot_mapping,
374 + block_tables=attn_metadata.block_tables)
375 +
376 + if frozen_model_input.seq_lens is not None:
377 + for i in range(num_queries):
378 + frozen_model_input.seq_lens[i] = attn_metadata.seq_lens[i]
379 +
380 + return model_input
381 +
382 + def load_model(self) -> None:
383 + return self._base_model_runner.load_model()
384 +
385 + def save_sharded_state(
386 + self,
387 + path: str,
388 + pattern: Optional[str] = None,
389 + max_size: Optional[int] = None,
390 + ) -> None:
391 + return self._base_model_runner.save_sharded_state(
392 + path, pattern, max_size)
393 +
394 + def save_tensorized_model(self,
395 + tensorizer_config: TensorizerConfig) -> None:
396 + return self._base_model_runner.save_tensorized_model(tensorizer_config)
397 +
398 + def profile_run(self) -> None:
399 + return self._base_model_runner.profile_run()
400 +
401 + def remove_all_loras(self):
402 + return self._base_model_runner.remove_all_loras()
403 +
404 + def capture_model(self, kv_caches: List[List]) -> None:
405 + return self._base_model_runner.capture_model(kv_caches)
406 +
407 + @property
408 + def vocab_size(self) -> int:
409 + return self._base_model_runner.vocab_size
410 +
411 +
412 +def _pythonize_sampler_output(model_input: StatefulModelInput,
413 + output: SamplerOutput,
414 + pinned_sampled_token_buffer: torch.Tensor,
415 + sampled_token_ids: torch.Tensor) -> None:
416 + """ This function is only called when the output tensors are ready.
417 + See ModelOutput
418 + """
419 +
420 + assert model_input.frozen_model_input is not None
421 +
422 + frozen_model_input = model_input.frozen_model_input
423 + assert frozen_model_input.sampling_metadata is not None
424 + # samples generation should have been skipped
425 + assert not output.outputs
426 +
427 + pinned_buffer = pinned_sampled_token_buffer[:model_input.num_queries]
428 +
429 + # CPU GPU sync
430 + pinned_buffer = pinned_buffer.copy_(sampled_token_ids, non_blocking=False)
431 +
432 + # this will not block as the tensors are already on CPU
433 + samples_list = pinned_buffer.tolist()
434 +
435 + sampling_metadata = frozen_model_input.sampling_metadata
436 +
437 + for (seq_group, sample_result) in zip(sampling_metadata.seq_groups,
438 + samples_list):
439 + seq_ids = seq_group.seq_ids
440 + next_token_ids = sample_result
441 + parent_ids = [0]
442 + seq_outputs: List[SequenceOutput] = []
443 + if seq_group.sampling_params.logits_processors:
444 + assert len(seq_group.sampling_params.logits_processors) == 0, (
445 + "Logits Processors are not supported in multi-step decoding")
446 + for parent_id, next_token_id in zip(parent_ids, next_token_ids):
447 + # TODO(will): support logprobs
448 + # Hard coded logprob
449 + seq_outputs.append(
450 + SequenceOutput(seq_ids[parent_id], next_token_id,
451 + {next_token_id: Logprob(logprob=-1)}))
452 + output.outputs.append(CompletionSequenceGroupOutput(seq_outputs, None))
453 + assert len(output.outputs) > 0
vllm/worker/multi_step_worker.py new
+189
@@ -0,0 +1,189 @@
1 +from dataclasses import dataclass
2 +from typing import List, Optional, Tuple
3 +
4 +from vllm.distributed import broadcast_tensor_dict, get_pp_group
5 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
6 +from vllm.worker.model_runner_base import BroadcastableModelInput
7 +from vllm.worker.multi_step_model_runner import (MultiStepModelRunner,
8 + StatefulModelInput)
9 +from vllm.worker.worker import Worker, WorkerInput
10 +
11 +
12 +@dataclass
13 +class MultiStepState:
14 + worker_input: WorkerInput
15 + model_input: StatefulModelInput
16 +
17 +
18 +class MultiStepWorker(Worker):
19 +
20 + def __init__(self, *args, **kwargs):
21 + super().__init__(*args, **kwargs)
22 + base_model_runner = self.model_runner
23 + # for multi-step model, wrap the model runner with MultiStepModelRunner
24 + self.model_runner = MultiStepModelRunner(
25 + base_model_runner,
26 + base_model_runner.model_config,
27 + base_model_runner.parallel_config,
28 + base_model_runner.scheduler_config,
29 + base_model_runner.device_config,
30 + base_model_runner.cache_config,
31 + load_config=base_model_runner.load_config,
32 + lora_config=self.lora_config,
33 + kv_cache_dtype=self.cache_config.cache_dtype,
34 + is_driver_worker=base_model_runner.is_driver_worker,
35 + prompt_adapter_config=base_model_runner.prompt_adapter_config,
36 + observability_config=base_model_runner.observability_config,
37 + )
38 +
39 + pipeline_parallel_size = self.parallel_config.pipeline_parallel_size
40 + self.multi_step_states: List[
41 + Optional[MultiStepState]] = [None] * pipeline_parallel_size
42 + self.temp_output = None
43 +
44 + def _get_driver_input_and_broadcast(
45 + self, execute_model_req: ExecuteModelRequest
46 + ) -> Tuple[BroadcastableModelInput, WorkerInput]:
47 + """
48 + Get the driver input and broadcast it to other workers.
49 + """
50 + assert self.is_driver_worker
51 + virtual_engine = execute_model_req.virtual_engine
52 + is_first_multi_step = execute_model_req.is_first_multi_step
53 + if is_first_multi_step:
54 + # on first step we prepare the worker input and model input normally
55 + worker_input: WorkerInput = self.prepare_worker_input(
56 + execute_model_req=execute_model_req)
57 + model_input: StatefulModelInput = (
58 + self.model_runner.prepare_model_input(
59 + execute_model_req.seq_group_metadata_list,
60 + execute_model_req.virtual_engine,
61 + execute_model_req.finished_requests_ids))
62 + else:
63 + # on subsequent steps we reuse the worker input and model input
64 + multi_step_state = self.multi_step_states[virtual_engine]
65 + worker_input = multi_step_state.worker_input
66 + model_input = multi_step_state.model_input
67 + frozen_model_input = model_input.frozen_model_input
68 + assert frozen_model_input is not None
69 + assert frozen_model_input.attn_metadata is not None
70 + # clear the cached decode metadata so that it can be recomputed on
71 + # the workers
72 + frozen_model_input.attn_metadata._cached_decode_metadata = None
73 +
74 + model_input.is_first_multi_step = is_first_multi_step
75 + model_input.is_last_step = execute_model_req.is_last_step
76 +
77 + if not is_first_multi_step:
78 + # we broadcast the last sampled token ids to all TP workers so they
79 + # can update their model input metadata in-place.
80 + self._prepare_last_sampled_token_ids_for_tp_workers(
81 + execute_model_req=execute_model_req, model_input=model_input)
82 +
83 + if self.do_metadata_broadcast:
84 + broadcast_data = worker_input.as_broadcastable_tensor_dict()
85 + broadcast_data.update(model_input.as_broadcastable_tensor_dict())
86 + broadcast_tensor_dict(broadcast_data, src=0)
87 +
88 + return model_input, worker_input
89 +
90 + def _prepare_last_sampled_token_ids_for_tp_workers(
91 + self,
92 + execute_model_req: ExecuteModelRequest,
93 + model_input: StatefulModelInput,
94 + ) -> None:
95 + """
96 + Prepare the last sampled token ids for TP workers. If it's the last
97 + PP rank, then the last sampled token ids are already in the model_input.
98 + If it is NOT the last PP rank, then we need to get the last sampled
99 + token that is cached in the execute_model_req.
100 + """
101 + if get_pp_group().is_last_rank:
102 + assert model_input.cached_outputs[
103 + -1].sampler_output.sampled_token_ids is None
104 + assert model_input.cached_outputs[-1].sampled_token_ids is not None
105 + model_input.last_sampled_token_ids = model_input.cached_outputs[
106 + -1].sampled_token_ids
107 + # free sampled token ids from the previous step if it has been
108 + # pythonized. Cannot free the last sampled token ids because
109 + # we need it for GPU advance_step.
110 + for output in model_input.cached_outputs[:-1]:
111 + if output.pythonized:
112 + output.sampled_token_ids = None
113 + else:
114 + # otherwise we need to get the cached sampled token ids from the
115 + # execute_model_req
116 + assert execute_model_req.last_sampled_token_ids is not None
117 + model_input.last_sampled_token_ids = (
118 + execute_model_req.last_sampled_token_ids.cuda())
119 + model_input.add_sampler_output(
120 + SamplerOutput(outputs=[], sampled_token_ids=None),
121 + model_input.last_sampled_token_ids)
122 +
123 + # free sampled token ids from the previous step.
124 + # TODO(will) we could reuse the sampled token ids tensor from
125 + # the previous step instead.
126 + for output in model_input.cached_outputs[:-1]:
127 + output.sampled_token_ids = None
128 + assert model_input.cached_outputs[-1].sampled_token_ids is not None
129 +
130 + def prepare_input(
131 + self,
132 + execute_model_req: Optional[ExecuteModelRequest] = None,
133 + ) -> Optional[Tuple[StatefulModelInput, WorkerInput]]:
134 + """
135 + Depending on the current state of the request and multi step worker,
136 + this method may skip the normal _prepare_model_input and
137 + _prepare_worker_input methods and instead used cached values.
138 + """
139 + if self.is_driver_worker:
140 + if execute_model_req is None:
141 + if self.do_metadata_broadcast:
142 + # This signals that there's no more requests to process for
143 + # now. All workers are running infinite loop with
144 + # broadcast_tensor_dict, and it stops the loop when the
145 + # driver broadcasts an empty input. Send an empty input to
146 + # notify all other workers to stop their execution loop.
147 + broadcast_tensor_dict({}, src=0)
148 + return None
149 +
150 + virtual_engine = execute_model_req.virtual_engine
151 + model_input, worker_input = self._get_driver_input_and_broadcast(
152 + execute_model_req)
153 + assert isinstance(model_input, StatefulModelInput)
154 + if execute_model_req.is_first_multi_step:
155 + # cache the worker input and model input for the next steps
156 + self.multi_step_states[virtual_engine] = MultiStepState(
157 + worker_input=worker_input, model_input=model_input)
158 + # if TP workers
159 + else:
160 + broadcast_data = self._get_worker_input_from_broadcast()
161 + # if the driver has sent an empty input, we should stop the worker
162 + # loop
163 + if broadcast_data is None:
164 + return None
165 + model_input, worker_input = broadcast_data
166 + assert isinstance(model_input, StatefulModelInput)
167 + virtual_engine = worker_input.virtual_engine
168 + if model_input.is_first_multi_step:
169 + pass
170 + # TODO(will) Can cache the worker input and model input for the
171 + # next steps. See below for details
172 + else:
173 + # TODO(will) possible to also cache and reuse the cached worker
174 + # input and model input. The idea is essentially the delta
175 + # optimization for model_inputs. Where the TP workers can cache
176 + # the model input states and we only broadcast the delta need
177 + # for the next step (sampled_token_ids from the previous step)
178 +
179 + assert isinstance(model_input, StatefulModelInput)
180 + # we need to update the last sampled token ids in the model
181 + # input for the workers so that they can run inplace
182 + # advance_step
183 + model_input.add_sampler_output(
184 + SamplerOutput(outputs=[], sampled_token_ids=None),
185 + model_input.last_sampled_token_ids)
186 +
187 + assert model_input is not None
188 + assert worker_input is not None
189 + return model_input, worker_input
vllm/worker/worker_base.py
+6 -4
@@ -16,7 +16,9 @@ from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
16 SamplerOutput)
17 from vllm.utils import (enable_trace_function_call_for_thread,
18 update_environment_variables)
19 -from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
19 +from vllm.worker.model_runner_base import (BroadcastableModelInput,
20 + ModelRunnerBase,
21 + ModelRunnerInputBase)
22
23 logger = init_logger(__name__)
24
@@ -220,7 +222,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
222 raise NotImplementedError
223
224 def _get_worker_input_from_broadcast(
223 - self) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
225 + self) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
226 """ Get the worker input from the broadcasted tensor dict. """
227 assert self.do_metadata_broadcast
228 assert not self.is_driver_worker
@@ -237,7 +239,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
239
240 def _get_driver_input_and_broadcast(
241 self, execute_model_req: ExecuteModelRequest
240 - ) -> Tuple[ModelRunnerInputBase, WorkerInput]:
242 + ) -> Tuple[BroadcastableModelInput, WorkerInput]:
243 """ Get the driver input and broadcast it to other workers. """
244 assert self.is_driver_worker
245
@@ -259,7 +261,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
261 def prepare_input(
262 self,
263 execute_model_req: Optional[ExecuteModelRequest] = None
262 - ) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
264 + ) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
265 """
266 Prepare the inputs to ModelRunner and workers.
267 """