A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Asynchronous Output Processor (#7049)
Co-authored-by: Alexander Matveev <alexm@neuralmagic.com>
Megha Agarwal committed
Aug 26, 2024 at 20:53 UTC
2eedede87502be64f60962147513f3df6cb1bd01
21 files changed
+636
-198
benchmarks/benchmark_throughput.py
+9
-1
@@ -86,6 +86,7 @@ def run_vllm(
86
use_v2_block_manager: bool = False,
87
download_dir: Optional[str] = None,
88
load_format: str = EngineArgs.load_format,
89
+ disable_async_output_proc: bool = False,
90
) -> float:
91
from vllm import LLM, SamplingParams
92
llm = LLM(
@@ -110,6 +111,7 @@ def run_vllm(
111
load_format=load_format,
112
num_scheduler_steps=num_scheduler_steps,
113
use_v2_block_manager=use_v2_block_manager,
114
+ disable_async_output_proc=disable_async_output_proc,
115
)
116
117
# Add the requests to the engine.
@@ -237,7 +239,8 @@ def main(args: argparse.Namespace):
239
args.enable_prefix_caching, args.enable_chunked_prefill,
240
args.max_num_batched_tokens, args.distributed_executor_backend,
241
args.gpu_memory_utilization, args.num_scheduler_steps,
240
- args.use_v2_block_manager, args.download_dir, args.load_format)
242
+ args.use_v2_block_manager, args.download_dir, args.load_format,
243
+ args.disable_async_output_proc)
244
elif args.backend == "hf":
245
assert args.tensor_parallel_size == 1
246
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -418,6 +421,11 @@ if __name__ == "__main__":
421
'section for more information.\n'
422
'* "bitsandbytes" will load the weights using bitsandbytes '
423
'quantization.\n')
424
+ parser.add_argument(
425
+ "--disable-async-output-proc",
426
+ action='store_true',
427
+ default=False,
428
+ help="Disable async output processor for vLLM backend.")
429
args = parser.parse_args()
430
if args.tokenizer is None:
431
args.tokenizer = args.model
tests/basic_correctness/test_chunked_prefill.py
+6
@@ -88,6 +88,9 @@ def test_models(
88
# NOTE: Increasing this in this suite will fail CI because we currently cannot
89
# reset distributed env properly. Use a value > 1 just when you test.
90
@pytest.mark.parametrize("tensor_parallel_size", [1])
91
+# Due to low-precision numerical divergence, this test is too sensitive to
92
+# the async postprocessor
93
+@pytest.mark.parametrize("disable_async_output_proc", [True])
94
def test_models_with_fp8_kv_cache(
95
vllm_runner,
96
example_prompts,
@@ -97,6 +100,7 @@ def test_models_with_fp8_kv_cache(
100
chunked_prefill_token_size: int,
101
enforce_eager: bool,
102
tensor_parallel_size: int,
103
+ disable_async_output_proc: bool,
104
) -> None:
105
"""
106
Only checks log probs match between chunked-prefill and
@@ -126,6 +130,7 @@ def test_models_with_fp8_kv_cache(
130
enforce_eager=enforce_eager,
131
max_num_seqs=max_num_seqs,
132
kv_cache_dtype=kv_cache_dtype,
133
+ disable_async_output_proc=disable_async_output_proc,
134
**extra_kwargs,
135
) as vllm_model:
136
no_chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
@@ -139,6 +144,7 @@ def test_models_with_fp8_kv_cache(
144
enforce_eager=enforce_eager,
145
max_num_seqs=max_num_seqs,
146
kv_cache_dtype=kv_cache_dtype,
147
+ disable_async_output_proc=disable_async_output_proc,
148
**extra_kwargs,
149
) as vllm_model:
150
chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
tests/basic_correctness/test_preemption.py
-1
@@ -209,7 +209,6 @@ def test_swap_infeasible(
209
prefill_blocks = 2
210
decode_blocks = max_tokens // BLOCK_SIZE
211
example_prompts = example_prompts[:1]
212
-
212
with vllm_runner(
213
model,
214
dtype=dtype,
tests/core/test_chunked_prefill_scheduler.py
+2
-2
@@ -21,7 +21,7 @@ def append_new_token(seq_group, token_id: int):
21
22
23
def schedule_and_update_computed_tokens(scheduler):
24
- metas, out = scheduler.schedule()
24
+ metas, out, _ = scheduler.schedule()
25
for s, meta in zip(out.scheduled_seq_groups, metas):
26
s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
27
return metas, out
@@ -180,7 +180,7 @@ def test_maximal_decoding():
180
"""Verify decoding requests are prioritized."""
181
block_size = 4
182
max_seqs = 2
183
- max_model_len = 2
183
+ max_model_len = 8
184
max_num_batched_tokens = 2
185
scheduler_config = SchedulerConfig(max_num_batched_tokens,
186
max_seqs,
tests/core/utils.py
+1
-1
@@ -199,7 +199,7 @@ def append_new_token(out, token_id: int):
199
200
201
def schedule_and_update_computed_tokens(scheduler):
202
- metas, out = scheduler.schedule()
202
+ metas, out, _ = scheduler.schedule()
203
for s, meta in zip(out.scheduled_seq_groups, metas):
204
s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
205
return metas, out
tests/engine/test_stop_strings.py
+103
-52
@@ -7,6 +7,8 @@ from vllm import CompletionOutput, LLMEngine, SamplingParams
7
MODEL = "meta-llama/llama-2-7b-hf"
8
MAX_TOKENS = 200
9
10
+IS_ASYNC = False
11
+
12
13
@pytest.fixture(scope="session")
14
def vllm_model(vllm_runner):
@@ -14,99 +16,148 @@ def vllm_model(vllm_runner):
16
yield vllm_model
17
18
17
-@pytest.mark.skip_global_cleanup
18
-def test_stop_basic(vllm_model):
19
- _test_stopping(vllm_model.model.llm_engine,
19
+def _test_stopping(llm_engine: LLMEngine,
20
+ expected_output: str,
21
+ expected_reason: Any,
22
+ stop: Optional[List[str]] = None,
23
+ stop_token_ids: Optional[List[int]] = None,
24
+ include_in_output: bool = False,
25
+ use_async_output_proc: bool = False) -> None:
26
+ llm_engine.add_request(
27
+ "id", "A story about vLLM:\n",
28
+ SamplingParams(
29
+ temperature=0.0,
30
+ max_tokens=MAX_TOKENS,
31
+ stop=stop,
32
+ stop_token_ids=stop_token_ids,
33
+ include_stop_str_in_output=include_in_output,
34
+ ), None)
35
+
36
+ output: Optional[CompletionOutput] = None
37
+ output_text = ""
38
+ stop_reason = None
39
+
40
+ if use_async_output_proc:
41
+ llm_engine.step()
42
+
43
+ while llm_engine.has_unfinished_requests():
44
+ (request_output, ) = llm_engine.step()
45
+ (output, ) = request_output.outputs
46
+
47
+ # Ensure we don't backtrack
48
+ assert output.text.startswith(output_text)
49
+ output_text = output.text
50
+ stop_reason = output.stop_reason
51
+
52
+ assert output is not None
53
+ assert output_text == expected_output
54
+ assert stop_reason == expected_reason
55
+
56
+
57
+def _set_async_mode(llm_engine, is_async):
58
+ llm_engine.scheduler[0].use_async_output_proc = is_async
59
+
60
+
61
+def _stop_basic(llm_engine, is_async):
62
+ _test_stopping(llm_engine,
63
stop=["."],
64
include_in_output=False,
65
expected_output="VLLM is a 100% volunteer organization",
23
- expected_reason=".")
66
+ expected_reason=".",
67
+ use_async_output_proc=is_async)
68
25
- _test_stopping(vllm_model.model.llm_engine,
69
+ _test_stopping(llm_engine,
70
stop=["."],
71
include_in_output=True,
72
expected_output="VLLM is a 100% volunteer organization.",
29
- expected_reason=".")
73
+ expected_reason=".",
74
+ use_async_output_proc=is_async)
75
76
32
-@pytest.mark.skip_global_cleanup
33
-def test_stop_multi_tokens(vllm_model):
77
+def _stop_multi_tokens(llm_engine, is_async):
78
_test_stopping(
35
- vllm_model.model.llm_engine,
79
+ llm_engine,
80
stop=["group of peo", "short"],
81
include_in_output=False,
82
expected_output="VLLM is a 100% volunteer organization. We are a ",
39
- expected_reason="group of peo")
83
+ expected_reason="group of peo",
84
+ use_async_output_proc=is_async)
85
86
_test_stopping(
42
- vllm_model.model.llm_engine,
87
+ llm_engine,
88
stop=["group of peo", "short"],
89
include_in_output=True,
90
expected_output=
91
"VLLM is a 100% volunteer organization. We are a group of peo",
47
- expected_reason="group of peo")
92
+ expected_reason="group of peo",
93
+ use_async_output_proc=is_async)
94
95
50
-@pytest.mark.skip_global_cleanup
51
-def test_stop_partial_token(vllm_model):
52
- _test_stopping(vllm_model.model.llm_engine,
96
+def _stop_partial_token(llm_engine, is_async):
97
+ _test_stopping(llm_engine,
98
stop=["gani"],
99
include_in_output=False,
100
expected_output="VLLM is a 100% volunteer or",
56
- expected_reason="gani")
101
+ expected_reason="gani",
102
+ use_async_output_proc=is_async)
103
58
- _test_stopping(vllm_model.model.llm_engine,
104
+ _test_stopping(llm_engine,
105
stop=["gani"],
106
include_in_output=True,
107
expected_output="VLLM is a 100% volunteer organi",
62
- expected_reason="gani")
108
+ expected_reason="gani",
109
+ use_async_output_proc=is_async)
110
111
65
-@pytest.mark.skip_global_cleanup
66
-def test_stop_token_id(vllm_model):
112
+def _stop_token_id(llm_engine, is_async):
113
# token id 13013 => " organization"
114
69
- _test_stopping(vllm_model.model.llm_engine,
115
+ _test_stopping(llm_engine,
116
stop_token_ids=[13013],
117
include_in_output=False,
118
expected_output="VLLM is a 100% volunteer",
73
- expected_reason=13013)
119
+ expected_reason=13013,
120
+ use_async_output_proc=is_async)
121
75
- _test_stopping(vllm_model.model.llm_engine,
122
+ _test_stopping(llm_engine,
123
stop_token_ids=[13013],
124
include_in_output=True,
125
expected_output="VLLM is a 100% volunteer organization",
79
- expected_reason=13013)
126
+ expected_reason=13013,
127
+ use_async_output_proc=is_async)
128
129
82
-def _test_stopping(llm_engine: LLMEngine,
83
- expected_output: str,
84
- expected_reason: Any,
85
- stop: Optional[List[str]] = None,
86
- stop_token_ids: Optional[List[int]] = None,
87
- include_in_output: bool = False) -> None:
88
- llm_engine.add_request(
89
- "id", "A story about vLLM:\n",
90
- SamplingParams(
91
- temperature=0.0,
92
- max_tokens=MAX_TOKENS,
93
- stop=stop,
94
- stop_token_ids=stop_token_ids,
95
- include_stop_str_in_output=include_in_output,
96
- ), None)
130
+@pytest.mark.skip_global_cleanup
131
+def test_stop_basic(vllm_model):
132
+ _set_async_mode(vllm_model.model.llm_engine, True)
133
+ _stop_basic(vllm_model.model.llm_engine, is_async=True)
134
98
- output: Optional[CompletionOutput] = None
99
- output_text = ""
100
- stop_reason = None
101
- while llm_engine.has_unfinished_requests():
102
- (request_output, ) = llm_engine.step()
103
- (output, ) = request_output.outputs
135
+ _set_async_mode(vllm_model.model.llm_engine, False)
136
+ _stop_basic(vllm_model.model.llm_engine, is_async=False)
137
105
- # Ensure we don't backtrack
106
- assert output.text.startswith(output_text)
107
- output_text = output.text
108
- stop_reason = output.stop_reason
138
110
- assert output is not None
111
- assert output_text == expected_output
112
- assert stop_reason == expected_reason
139
+@pytest.mark.skip_global_cleanup
140
+def test_stop_multi_tokens(vllm_model):
141
+ _set_async_mode(vllm_model.model.llm_engine, True)
142
+ _stop_multi_tokens(vllm_model.model.llm_engine, is_async=True)
143
+
144
+ _set_async_mode(vllm_model.model.llm_engine, False)
145
+ _stop_multi_tokens(vllm_model.model.llm_engine, is_async=False)
146
+
147
+
148
+@pytest.mark.skip_global_cleanup
149
+def test_stop_partial_token(vllm_model):
150
+ _set_async_mode(vllm_model.model.llm_engine, True)
151
+ _stop_partial_token(vllm_model.model.llm_engine, is_async=True)
152
+
153
+ _set_async_mode(vllm_model.model.llm_engine, False)
154
+ _stop_partial_token(vllm_model.model.llm_engine, is_async=False)
155
+
156
+
157
+@pytest.mark.skip_global_cleanup
158
+def test_stop_token_id(vllm_model):
159
+ _set_async_mode(vllm_model.model.llm_engine, True)
160
+ _stop_token_id(vllm_model.model.llm_engine, is_async=True)
161
+
162
+ _set_async_mode(vllm_model.model.llm_engine, False)
163
+ _stop_token_id(vllm_model.model.llm_engine, is_async=False)
tests/multi_step/test_correctness_async_llm.py
+3
@@ -62,6 +62,9 @@ async def test_multi_step(example_prompts, model: str, tp_size: int,
62
ms_server_args = DEFAULT_SERVER_ARGS + \
63
["--num-scheduler-steps", f"{num_scheduler_steps}"]
64
65
+ # Disable output proc callback as its not supported
66
+ # with multi-step right now
67
+ ms_server_args += ["--disable-async-output-proc"]
68
if eager_mode:
69
ms_server_args.append("--enforce-eager")
70
vllm/config.py
+53
@@ -140,6 +140,7 @@ class ModelConfig:
140
skip_tokenizer_init: bool = False,
141
served_model_name: Optional[Union[str, List[str]]] = None,
142
limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
143
+ use_async_output_proc: bool = True,
144
) -> None:
145
self.model = model
146
self.tokenizer = tokenizer
@@ -172,6 +173,7 @@ class ModelConfig:
173
self.hf_image_processor_config = get_hf_image_processor_config(
174
self.model, revision)
175
self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
176
+ self.use_async_output_proc = use_async_output_proc
177
178
# Choose a default enforce_eager value if the user did not specify
179
# a value (enforce_eager is None)
@@ -326,6 +328,49 @@ class ModelConfig:
328
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
329
self.max_model_len)
330
331
+ def verify_async_output_proc(self, parallel_config, speculative_config,
332
+ device_config) -> None:
333
+ if not self.use_async_output_proc:
334
+ # Nothing to check
335
+ return
336
+
337
+ if parallel_config.pipeline_parallel_size > 1:
338
+ logger.warning("Async output processing can not be enabled "
339
+ "with pipeline parallel")
340
+ self.use_async_output_proc = False
341
+ return
342
+
343
+ if device_config.device_type != "cuda":
344
+ logger.warning(
345
+ "Async output processing is only supported for CUDA."
346
+ " Disabling it for other platforms.")
347
+ self.use_async_output_proc = False
348
+ return
349
+
350
+ if envs.VLLM_USE_RAY_SPMD_WORKER:
351
+ logger.warning(
352
+ "Async output processing can not be enabled with ray spmd")
353
+ self.use_async_output_proc = False
354
+ return
355
+
356
+ if self.enforce_eager:
357
+ logger.warning(
358
+ "To see benefits of async output processing, enable CUDA "
359
+ "graph. Since, enforce-eager is enabled, async output "
360
+ "processor cannot be used")
361
+ self.use_async_output_proc = not self.enforce_eager
362
+ return
363
+
364
+ # Async postprocessor is not necessary with embedding mode
365
+ # since there is no token generation
366
+ if self.embedding_mode:
367
+ self.use_async_output_proc = False
368
+
369
+ if speculative_config:
370
+ logger.warning("Async output processing is not supported with"
371
+ " speculative decoding currently.")
372
+ self.use_async_output_proc = False
373
+
374
def verify_with_parallel_config(
375
self,
376
parallel_config: "ParallelConfig",
@@ -358,6 +403,11 @@ class ModelConfig:
403
"fallback to the eager mode.")
404
self.enforce_eager = True
405
406
+ if pipeline_parallel_size > 1 and self.use_async_output_proc:
407
+ logger.warning("Async output processor is not supported with "
408
+ "pipeline parallelism currently. Disabling it.")
409
+ self.use_async_output_proc = False
410
+
411
def get_hf_config_sliding_window(self) -> Optional[int]:
412
"""Get the sliding window size, or None if disabled."""
413
@@ -1769,6 +1819,9 @@ class EngineConfig:
1819
def __post_init__(self):
1820
"""Verify configs are valid & consistent with each other.
1821
"""
1822
+ self.model_config.verify_async_output_proc(self.parallel_config,
1823
+ self.speculative_config,
1824
+ self.device_config)
1825
self.model_config.verify_with_parallel_config(self.parallel_config)
1826
self.cache_config.verify_with_parallel_config(self.parallel_config)
1827
vllm/core/scheduler.py
+118
-12
@@ -4,7 +4,8 @@ import random
4
import time
5
from collections import deque
6
from dataclasses import dataclass, field
7
-from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
7
+from typing import (Callable, Deque, Dict, Iterable, List, Optional, Set,
8
+ Tuple, Union)
9
10
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
11
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
@@ -299,6 +300,7 @@ class Scheduler:
300
cache_config: CacheConfig,
301
lora_config: Optional[LoRAConfig],
302
pipeline_parallel_size: int = 1,
303
+ output_proc_callback_fn: Optional[Callable] = None,
304
) -> None:
305
self.scheduler_config = scheduler_config
306
self.cache_config = cache_config
@@ -364,10 +366,36 @@ class Scheduler:
366
self.num_cumulative_preemption: int = 0
367
368
# Used to cache python objects
367
- self._scheduler_running_outputs_cache: PyObjectCache = PyObjectCache(
368
- scheduler_running_outputs_builder)
369
- self._scheduled_seq_group_cache: PyObjectCache = PyObjectCache(
370
- scheduled_seq_group_builder)
369
+ self._seq_group_metadata_cache: List[PyObjectCache] = []
370
+ self._scheduler_running_outputs_cache: List[PyObjectCache] = []
371
+ self._scheduled_seq_group_cache: List[PyObjectCache] = []
372
+
373
+ # For async output processing, we need to swap cache buffers between
374
+ # iterations. I.e. since the output processing is lagged one step,
375
+ # we cannot reuse the cached objects immediately when the schedule()
376
+ # is called again, but only when schedule() is called the second time.
377
+ self.output_proc_callback_fn = output_proc_callback_fn
378
+ self.use_async_output_proc = self.output_proc_callback_fn is not None
379
+ self.num_cache_iters = 2 if self.use_async_output_proc else 1
380
+
381
+ self.cache_id = 0
382
+ for i in range(self.num_cache_iters):
383
+ self._seq_group_metadata_cache.append(
384
+ PyObjectCache(seq_group_metadata_builder))
385
+ self._scheduler_running_outputs_cache.append(
386
+ PyObjectCache(scheduler_running_outputs_builder))
387
+ self._scheduled_seq_group_cache.append(
388
+ PyObjectCache(scheduled_seq_group_builder))
389
+
390
+ # For async postprocessor, the extra decode run cannot be done
391
+ # when the request reaches max_model_len. In this case, the request
392
+ # will be stopped during schedule() call and added to this stop list
393
+ # for processing and deallocation by the free_finished_seq_groups()
394
+ self._async_stopped: List[SequenceGroup] = []
395
+
396
+ @property
397
+ def next_cache_id(self):
398
+ return (self.cache_id + 1) % self.num_cache_iters
399
400
@property
401
def lora_enabled(self) -> bool:
@@ -483,7 +511,7 @@ class Scheduler:
511
SchedulerRunningOutputs.
512
"""
513
ret: SchedulerRunningOutputs = \
486
- self._scheduler_running_outputs_cache.get_object()
514
+ self._scheduler_running_outputs_cache[self.cache_id].get_object()
515
ret.blocks_to_swap_out.clear()
516
ret.blocks_to_copy.clear()
517
ret.decode_seq_groups.clear()
@@ -510,8 +538,12 @@ class Scheduler:
538
# NOTE(woosuk): Preemption happens only when there is no available slot
539
# to keep all the sequence groups in the RUNNING state.
540
513
- running_queue = self.running
541
+ # Store original running requests for the case of async + preemption
542
+ if self.use_async_output_proc:
543
+ orig_running = self.running.copy()
544
545
+ running_queue = self.running
546
+ assert len(self._async_stopped) == 0
547
while running_queue:
548
seq_group = running_queue[0]
549
num_running_tokens = self._get_num_new_tokens(
@@ -521,6 +553,28 @@ class Scheduler:
553
break
554
555
running_queue.popleft()
556
+
557
+ # With async postprocessor, an extra decode run is done
558
+ # to process the final tokens. The check below avoids this extra
559
+ # decode run when the model max len is reached, in order to avoid
560
+ # a memory overflow.
561
+ if self.use_async_output_proc and seq_group.seqs[0].get_len(
562
+ ) > self.scheduler_config.max_model_len:
563
+ self._async_stopped.append(seq_group)
564
+ continue
565
+
566
+ # With async postprocessor, when preemption kicks in, we need
567
+ # first to drain the async postprocessor, so that all async
568
+ # block_table freeing is applied before the preemption freeing
569
+ # is applied.
570
+ if self.use_async_output_proc and not self._can_append_slots(
571
+ seq_group):
572
+ tmp = self.running
573
+ self.running = orig_running
574
+ assert self.output_proc_callback_fn is not None
575
+ self.output_proc_callback_fn(is_async=True)
576
+ self.running = tmp
577
+
578
while not self._can_append_slots(seq_group):
579
budget.subtract_num_batched_tokens(seq_group.request_id,
580
num_running_tokens)
@@ -556,7 +610,7 @@ class Scheduler:
610
is_prefill = seq_group.is_prefill()
611
612
scheduled_seq_group: ScheduledSequenceGroup = \
559
- self._scheduled_seq_group_cache.get_object()
613
+ self._scheduled_seq_group_cache[self.cache_id].get_object()
614
scheduled_seq_group.seq_group = seq_group
615
if is_prefill:
616
scheduled_seq_group.token_chunk_size = num_running_tokens
@@ -579,8 +633,8 @@ class Scheduler:
633
if curr_loras is not None and seq_group.lora_int_id > 0:
634
curr_loras.add(seq_group.lora_int_id)
635
582
- self._scheduler_running_outputs_cache.reset()
583
- self._scheduled_seq_group_cache.reset()
636
+ self._scheduler_running_outputs_cache[self.next_cache_id].reset()
637
+ self._scheduled_seq_group_cache[self.next_cache_id].reset()
638
639
return ret
640
@@ -1031,17 +1085,31 @@ class Scheduler:
1085
num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
1086
)
1087
1034
- def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
1088
+ def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1089
+ no_beam_search = (seq_group.sampling_params.best_of == 1
1090
+ and not seq_group.sampling_params.use_beam_search)
1091
+
1092
+ return no_beam_search
1093
+
1094
+ def schedule(
1095
+ self
1096
+ ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, bool]:
1097
# Schedule sequence groups.
1098
# This function call changes the internal states of the scheduler
1099
# such as self.running, self.swapped, and self.waiting.
1100
scheduler_start_time = time.perf_counter()
1101
+
1102
scheduler_outputs = self._schedule()
1103
now = time.time()
1104
1105
if not self.cache_config.enable_prefix_caching:
1106
common_computed_block_nums = []
1107
1108
+ # TODO: Combine multi-step and async postprocessor
1109
+ allow_async_output_proc: bool = (
1110
+ self.use_async_output_proc
1111
+ and not self.scheduler_config.is_multi_step)
1112
+
1113
# Create input data structures.
1114
seq_group_metadata_list: List[SequenceGroupMetadata] = []
1115
for i, scheduled_seq_group in enumerate(
@@ -1050,6 +1118,11 @@ class Scheduler:
1118
token_chunk_size = scheduled_seq_group.token_chunk_size
1119
seq_group.maybe_set_first_scheduled_time(now)
1120
1121
+ seq_group_metadata = self._seq_group_metadata_cache[
1122
+ self.cache_id].get_object()
1123
+ seq_group_metadata.seq_data.clear()
1124
+ seq_group_metadata.block_tables.clear()
1125
+
1126
# seq_id -> SequenceData
1127
seq_data: Dict[int, SequenceData] = {}
1128
# seq_id -> physical block numbers
@@ -1139,6 +1212,10 @@ class Scheduler:
1212
)
1213
seq_group_metadata_list.append(seq_group_metadata)
1214
1215
+ if allow_async_output_proc:
1216
+ allow_async_output_proc = self._allow_async_output_proc(
1217
+ seq_group)
1218
+
1219
# Now that the batch has been created, we can assume all blocks in the
1220
# batch will have been computed before the next scheduling invocation.
1221
# This is because the engine assumes that a failure in model execution
@@ -1147,6 +1224,8 @@ class Scheduler:
1224
self.block_manager.mark_blocks_as_computed(
1225
scheduled_seq_group.seq_group)
1226
1227
+ self._seq_group_metadata_cache[self.next_cache_id].reset()
1228
+
1229
scheduler_time = time.perf_counter() - scheduler_start_time
1230
# Add this to scheduler time to all the sequences that are currently
1231
# running. This will help estimate if the scheduler is a significant
@@ -1158,7 +1237,12 @@ class Scheduler:
1237
else:
1238
seq_group.metrics.scheduler_time = scheduler_time
1239
1161
- return seq_group_metadata_list, scheduler_outputs
1240
+ # Move to next cache (if exists)
1241
+ self.cache_id = self.next_cache_id
1242
+
1243
+ # Return results
1244
+ return (seq_group_metadata_list, scheduler_outputs,
1245
+ allow_async_output_proc)
1246
1247
def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
1248
self.block_manager.fork(parent_seq, child_seq)
@@ -1167,6 +1251,12 @@ class Scheduler:
1251
"""Free a sequence from a block table."""
1252
self.block_manager.free(seq)
1253
1254
+ def _free_finished_seqs(self, seq_group: SequenceGroup) -> None:
1255
+ """Free finished seqs in a sequence group."""
1256
+ for seq in seq_group.get_seqs():
1257
+ if seq.is_finished():
1258
+ self.free_seq(seq)
1259
+
1260
def free_finished_seq_groups(self) -> None:
1261
remaining: Deque[SequenceGroup] = deque()
1262
for seq_group in self.running:
@@ -1179,8 +1269,24 @@ class Scheduler:
1269
self._finished_requests_ids.append(seq_group.request_id)
1270
else:
1271
remaining.append(seq_group)
1272
+
1273
+ # Free finished seqs
1274
+ self._free_finished_seqs(seq_group)
1275
+
1276
self.running = remaining
1277
1278
+ # Handle async stopped sequence groups
1279
+ # (ones that reached max model len)
1280
+ if self._async_stopped:
1281
+ for seq_group in self._async_stopped:
1282
+ self._free_seq_group_cross_attn_blocks(seq_group)
1283
+ self._finished_requests_ids.append(seq_group.request_id)
1284
+
1285
+ # Free finished seqs
1286
+ self._free_finished_seqs(seq_group)
1287
+
1288
+ self._async_stopped.clear()
1289
+
1290
def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
1291
self.block_manager.allocate(seq_group)
1292
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
vllm/engine/arg_utils.py
+8
@@ -147,6 +147,7 @@ class EngineArgs:
147
148
otlp_traces_endpoint: Optional[str] = None
149
collect_detailed_traces: Optional[str] = None
150
+ disable_async_output_proc: bool = False
151
152
def __post_init__(self):
153
if self.tokenizer is None:
@@ -733,6 +734,12 @@ class EngineArgs:
734
"modules. This involves use of possibly costly and or blocking "
735
"operations and hence might have a performance impact.")
736
737
+ parser.add_argument(
738
+ '--disable-async-output-proc',
739
+ action='store_true',
740
+ default=EngineArgs.disable_async_output_proc,
741
+ help="Disable async output processing. This may result in "
742
+ "lower performance.")
743
return parser
744
745
@classmethod
@@ -792,6 +799,7 @@ class EngineArgs:
799
skip_tokenizer_init=self.skip_tokenizer_init,
800
served_model_name=self.served_model_name,
801
limit_mm_per_prompt=self.limit_mm_per_prompt,
802
+ use_async_output_proc=not self.disable_async_output_proc,
803
)
804
cache_config = CacheConfig(
805
block_size=self.block_size if self.device != "neuron" else
vllm/engine/async_llm_engine.py
+47
-13
@@ -277,23 +277,36 @@ class _AsyncLLMEngine(LLMEngine):
277
cached_outputs = self.cached_scheduler_outputs[virtual_engine]
278
seq_group_metadata_list = cached_outputs.seq_group_metadata_list
279
scheduler_outputs = cached_outputs.scheduler_outputs
280
+ allow_async_output_proc = cached_outputs.allow_async_output_proc
281
+
282
# skip the scheduler if there are any remaining steps in the seq groups.
283
# This ensures that the scheduler is only called again when the current
284
# batch has completed.
285
if not self._has_remaining_steps(seq_group_metadata_list):
284
- seq_group_metadata_list, scheduler_outputs = self.scheduler[
285
- virtual_engine].schedule()
286
+ (seq_group_metadata_list, scheduler_outputs,
287
+ allow_async_output_proc
288
+ ) = self.scheduler[virtual_engine].schedule()
289
+
290
+ # If current scheduler iteration has no async postprocessor,
291
+ # then we need first to drain the pending async postprocessor
292
+ # before moving forward
293
+ if not allow_async_output_proc and len(self.output_queue) > 0:
294
+ self._process_model_outputs(is_async=True)
295
296
if (self.scheduler_config.is_multi_step
297
and scheduler_outputs.num_lookahead_slots > 0):
298
# cache the scheduler outputs for the next iteration if we have
299
# lookahead slots
300
self._cache_scheduler_outputs_for_multi_step(
292
- virtual_engine, seq_group_metadata_list, scheduler_outputs)
301
+ virtual_engine, seq_group_metadata_list, scheduler_outputs,
302
+ allow_async_output_proc)
303
304
assert seq_group_metadata_list is not None
305
assert scheduler_outputs is not None
306
307
+ assert not (self.scheduler_config.is_multi_step and \
308
+ allow_async_output_proc)
309
+
310
if not scheduler_outputs.is_empty():
311
finished_requests_ids = self.scheduler[
312
virtual_engine].get_and_reset_finished_requests_ids()
@@ -317,6 +330,11 @@ class _AsyncLLMEngine(LLMEngine):
330
# We use ExecuteModelRequest to pass the last sampled_token_ids
331
# to each of the non-last PP stages for in-place prepare_input.
332
last_sampled_token_ids=last_sampled_token_ids)
333
+
334
+ if allow_async_output_proc:
335
+ execute_model_req.output_proc_callback_fn = \
336
+ self._process_model_outputs
337
+
338
# Execute the model.
339
output = await self.model_executor.execute_model_async(
340
execute_model_req)
@@ -325,6 +343,9 @@ class _AsyncLLMEngine(LLMEngine):
343
if self.scheduler_config.is_multi_step:
344
self._update_cached_scheduler_output(virtual_engine, output)
345
else:
346
+ if len(self.output_queue) > 0:
347
+ assert not self.scheduler_config.is_multi_step
348
+ self._process_model_outputs(is_async=True)
349
output = []
350
351
# Finish the current step for all the sequence groups.
@@ -337,19 +358,32 @@ class _AsyncLLMEngine(LLMEngine):
358
if self.scheduler_config.is_multi_step:
359
self.cached_scheduler_outputs[
360
virtual_engine] = SchedulerOutputState()
340
- request_outputs = self._process_model_outputs(
341
- output, scheduler_outputs.scheduled_seq_groups,
342
- scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
343
- else:
344
- request_outputs = []
361
346
- # Log stats.
347
- self.do_log_stats(scheduler_outputs, output)
362
+ # Cache results in engine
363
+ self.output_queue.append(
364
+ (output, seq_group_metadata_list, scheduler_outputs))
365
349
- # Tracing
350
- self.do_tracing(scheduler_outputs)
366
+ if output and allow_async_output_proc:
367
+ assert len(
368
+ output
369
+ ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
370
+ self._advance_to_next_step(
371
+ output[0], seq_group_metadata_list,
372
+ scheduler_outputs.scheduled_seq_groups)
373
+
374
+ if not allow_async_output_proc:
375
+ self._process_model_outputs(is_async=False)
376
+
377
+ # Log stats.
378
+ self.do_log_stats(scheduler_outputs, output)
379
+
380
+ # Tracing
381
+ self.do_tracing(scheduler_outputs)
382
+
383
+ else:
384
+ self.request_outputs = []
385
352
- return request_outputs
386
+ return self.request_outputs
387
388
async def stop_remote_worker_execution_loop_async(self) -> None:
389
"""Stop the remote worker execution loop."""
vllm/engine/llm_engine.py
+208
-70
@@ -1,7 +1,8 @@
1
import time
2
+from collections import deque
3
from contextlib import contextmanager
4
from dataclasses import dataclass
4
-from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5
+from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
6
Mapping, Optional)
7
from typing import Sequence as GenericSequence
8
from typing import Set, Tuple, Type, Union
@@ -38,9 +39,8 @@ from vllm.pooling_params import PoolingParams
39
from vllm.prompt_adapter.request import PromptAdapterRequest
40
from vllm.sampling_params import SamplingParams
41
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
41
- PoolerOutput, SamplerOutput, Sequence,
42
- SequenceGroup, SequenceGroupMetadata,
43
- SequenceStatus)
42
+ SamplerOutput, Sequence, SequenceGroup,
43
+ SequenceGroupMetadata, SequenceStatus)
44
from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
45
init_tracer)
46
from vllm.transformers_utils.config import try_get_generation_config
@@ -82,9 +82,10 @@ DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
82
@dataclass
83
class SchedulerOutputState:
84
"""Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
85
- last_output: Optional[SamplerOutput] = None
85
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
86
scheduler_outputs: Optional[SchedulerOutputs] = None
87
+ allow_async_output_proc: bool = False
88
+ last_output: Optional[SamplerOutput] = None
89
90
91
class LLMEngine:
@@ -190,6 +191,9 @@ class LLMEngine:
191
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
192
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
193
input_registry: InputRegistry = INPUT_REGISTRY,
194
+ # To improve performance, only final requests outputs may be required.
195
+ # If this set to true, then no intermediate outputs will be returned.
196
+ step_return_finished_only: bool = False,
197
) -> None:
198
logger.info(
199
"Initializing an LLM engine (v%s) with config: "
@@ -204,7 +208,8 @@ class LLMEngine:
208
"quantization_param_path=%s, device_config=%s, "
209
"decoding_config=%r, observability_config=%r, "
210
"seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
207
- "num_scheduler_steps=%d, enable_prefix_caching=%s)",
211
+ "num_scheduler_steps=%d, enable_prefix_caching=%s, "
212
+ "use_async_output_proc=%s)",
213
VLLM_VERSION,
214
model_config.model,
215
speculative_config,
@@ -235,6 +240,7 @@ class LLMEngine:
240
scheduler_config.use_v2_block_manager,
241
scheduler_config.num_scheduler_steps,
242
cache_config.enable_prefix_caching,
243
+ model_config.use_async_output_proc,
244
)
245
# TODO(woosuk): Print more configs in debug mode.
246
from vllm.plugins import load_general_plugins
@@ -253,6 +259,7 @@ class LLMEngine:
259
self.observability_config = observability_config or ObservabilityConfig(
260
)
261
self.log_stats = log_stats
262
+ self.step_return_finished_only = step_return_finished_only
263
264
if not self.model_config.skip_tokenizer_init:
265
self.tokenizer = self._init_tokenizer()
@@ -340,8 +347,11 @@ class LLMEngine:
347
# NOTE: the cache_config here have been updated with the numbers of
348
# GPU and CPU blocks, which are profiled in the distributed executor.
349
self.scheduler = [
343
- Scheduler(scheduler_config, cache_config, lora_config,
344
- parallel_config.pipeline_parallel_size)
350
+ Scheduler(
351
+ scheduler_config, cache_config, lora_config,
352
+ parallel_config.pipeline_parallel_size,
353
+ self._process_model_outputs
354
+ if model_config.use_async_output_proc else None)
355
for _ in range(parallel_config.pipeline_parallel_size)
356
]
357
@@ -396,6 +406,13 @@ class LLMEngine:
406
for _ in range(self.parallel_config.pipeline_parallel_size)
407
]
408
409
+ # Async output processing pointers
410
+ self.output_queue: Deque[Tuple[List[SamplerOutput],
411
+ List[SequenceGroupMetadata],
412
+ SchedulerOutputs]] = deque()
413
+ self.request_outputs: List[Union[RequestOutput,
414
+ EmbeddingRequestOutput]] = []
415
+
416
def _initialize_kv_caches(self) -> None:
417
"""Initialize the KV cache in the worker(s).
418
@@ -1197,34 +1214,66 @@ class LLMEngine:
1214
1215
return
1216
1200
- def _process_model_outputs(
1201
- self,
1202
- output: GenericSequence[Union[SamplerOutput, PoolerOutput]],
1203
- scheduled_seq_groups: List[ScheduledSequenceGroup],
1204
- ignored_seq_groups: List[SequenceGroup],
1205
- seq_group_metadata_list: List[SequenceGroupMetadata],
1206
- ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1217
+ def _process_model_outputs(self,
1218
+ is_async: bool,
1219
+ clear_outputs: bool = True) -> None:
1220
"""Apply the model output to the sequences in the scheduled seq groups.
1221
1222
+ is_async: Indicates whether this postprocessor runs in
1223
+ parallel with the GPU forward pass and is processing
1224
+ tokens from the previous step. If this is true, then
1225
+ no tokens need to be appended since it is already done
1226
+ externally (before the next schedule() call)
1227
+ clear_outputs: Sometimes existing outputs need to be combined
1228
+ with outputs of this call. This happens for postprocessor
1229
+ draining at the final stage (like when sequences are finished)
1230
+
1231
Returns RequestOutputs that can be returned to the client.
1232
"""
1211
-
1233
now = time.time()
1234
1214
- # Organize outputs by [sequence group][step] instead of
1215
- # [step][sequence group].
1216
- output_by_sequence_group = create_output_by_sequence_group(
1217
- output, num_seq_groups=len(scheduled_seq_groups))
1235
+ if clear_outputs:
1236
+ self.request_outputs.clear()
1237
+
1238
+ if len(self.output_queue) == 0:
1239
+ return None
1240
+
1241
+ (outputs, seq_group_metadata_list,
1242
+ scheduler_outputs) = self.output_queue.popleft()
1243
+
1244
+ # Sanity check
1245
+ assert len(seq_group_metadata_list) == len(
1246
+ scheduler_outputs.scheduled_seq_groups)
1247
+
1248
+ # Organize outputs by [step][sequence group] instead of
1249
+ # [sequence group][step].
1250
+ if len(outputs) > 1:
1251
+ outputs_by_sequence_group = create_output_by_sequence_group(
1252
+ outputs, num_seq_groups=len(seq_group_metadata_list))
1253
+ else:
1254
+ outputs_by_sequence_group = outputs
1255
+
1256
+ finished_before: List[int] = []
1257
+ for i, seq_group_meta in enumerate(seq_group_metadata_list):
1258
+ scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1259
1219
- # Update the scheduled sequence groups with the model outputs.
1220
- for scheduled_seq_group, outputs, seq_group_meta in zip(
1221
- scheduled_seq_groups, output_by_sequence_group,
1222
- seq_group_metadata_list):
1260
seq_group = scheduled_seq_group.seq_group
1224
- seq_group.update_num_computed_tokens(
1225
- scheduled_seq_group.token_chunk_size)
1226
- if output is not None and len(output) > 0:
1227
- for o in output:
1261
+
1262
+ if seq_group.is_finished():
1263
+ finished_before.append(i)
1264
+ continue
1265
+
1266
+ if len(outputs) > 1:
1267
+ output = outputs_by_sequence_group[i]
1268
+ else:
1269
+ output = [outputs_by_sequence_group[0][i]]
1270
+
1271
+ if not is_async:
1272
+ seq_group.update_num_computed_tokens(
1273
+ scheduled_seq_group.token_chunk_size)
1274
+
1275
+ if outputs:
1276
+ for o in outputs:
1277
if (isinstance(o, SamplerOutput)
1278
and seq_group.metrics is not None):
1279
if seq_group.metrics.model_forward_time is not None:
@@ -1239,30 +1288,75 @@ class LLMEngine:
1288
else:
1289
seq_group.metrics.model_execute_time = (
1290
o.model_execute_time)
1291
+
1292
if self.model_config.embedding_mode:
1243
- self._process_sequence_group_outputs(seq_group, outputs)
1293
+ self._process_sequence_group_outputs(seq_group, output)
1294
continue
1295
1246
- self.output_processor.process_prompt_logprob(seq_group, outputs)
1296
+ self.output_processor.process_prompt_logprob(seq_group, output)
1297
if seq_group_meta.do_sample:
1248
- self.output_processor.process_outputs(seq_group, outputs)
1298
+ self.output_processor.process_outputs(seq_group, output,
1299
+ is_async)
1300
1301
# Free the finished sequence groups.
1302
for scheduler in self.scheduler:
1303
scheduler.free_finished_seq_groups()
1304
1305
# Create the outputs.
1255
- request_outputs: List[Union[RequestOutput,
1256
- EmbeddingRequestOutput]] = []
1257
- for scheduled_seq_group in scheduled_seq_groups:
1306
+ for i, _ in enumerate(seq_group_metadata_list):
1307
+ scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1308
+
1309
+ if i in finished_before:
1310
+ continue # Avoids double processing
1311
+
1312
seq_group = scheduled_seq_group.seq_group
1313
seq_group.maybe_set_first_token_time(now)
1314
+ if (seq_group.is_finished()
1315
+ if self.step_return_finished_only else True):
1316
+ request_output = RequestOutputFactory.create(seq_group)
1317
+ self.request_outputs.append(request_output)
1318
+
1319
+ for seq_group in scheduler_outputs.ignored_seq_groups:
1320
request_output = RequestOutputFactory.create(seq_group)
1261
- request_outputs.append(request_output)
1262
- for seq_group in ignored_seq_groups:
1263
- request_output = RequestOutputFactory.create(seq_group)
1264
- request_outputs.append(request_output)
1265
- return request_outputs
1321
+ self.request_outputs.append(request_output)
1322
+
1323
+ if is_async:
1324
+ # Log stats.
1325
+ self.do_log_stats(scheduler_outputs, outputs, finished_before)
1326
+
1327
+ # Tracing
1328
+ self.do_tracing(scheduler_outputs)
1329
+
1330
+ return None
1331
+
1332
+ def _advance_to_next_step(
1333
+ self, output: List[SamplerOutput],
1334
+ seq_group_metadata_list: List[SequenceGroupMetadata],
1335
+ scheduled_seq_groups: List[ScheduledSequenceGroup]) -> None:
1336
+ """Given model output from a single run, append the tokens to the
1337
+ sequences. This is normally done inside output processor, but it is
1338
+ required if the worker is to perform async forward pass to next step.
1339
+ """
1340
+ for seq_group_metadata, sequence_group_outputs, scheduled_seq_group in \
1341
+ zip(seq_group_metadata_list, output, scheduled_seq_groups):
1342
+ seq_group = scheduled_seq_group.seq_group
1343
+
1344
+ if seq_group.is_finished():
1345
+ continue
1346
+
1347
+ seq_group.update_num_computed_tokens(
1348
+ seq_group_metadata.token_chunk_size)
1349
+
1350
+ if seq_group_metadata.do_sample:
1351
+ assert len(sequence_group_outputs.samples) == 1, (
1352
+ "Async output processor expects a single sample"
1353
+ " (i.e sampling_params.n == 1 and no "
1354
+ "sampling_params.best_of > 1)")
1355
+ sample = sequence_group_outputs.samples[0]
1356
+
1357
+ assert len(seq_group.seqs) == 1
1358
+ seq = seq_group.seqs[0]
1359
+ seq.append_token_id(sample.output_token, sample.logprobs)
1360
1361
def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1362
"""Performs one decoding iteration and returns newly generated results.
@@ -1325,24 +1419,32 @@ class LLMEngine:
1419
cached_outputs = self.cached_scheduler_outputs[0]
1420
seq_group_metadata_list = cached_outputs.seq_group_metadata_list
1421
scheduler_outputs = cached_outputs.scheduler_outputs
1422
+ allow_async_output_proc = cached_outputs.allow_async_output_proc
1423
1424
# Skip the scheduler if there are any remaining steps in the seq groups.
1425
# This ensures that the scheduler is only called again when the current
1426
# batch has completed.
1427
if not self._has_remaining_steps(seq_group_metadata_list):
1333
- seq_group_metadata_list, scheduler_outputs = self.scheduler[
1334
- 0].schedule()
1428
+ (seq_group_metadata_list, scheduler_outputs,
1429
+ allow_async_output_proc) = self.scheduler[0].schedule()
1430
+
1431
+ if not allow_async_output_proc and len(self.output_queue) > 0:
1432
+ self._process_model_outputs(is_async=True)
1433
1434
if (self.scheduler_config.is_multi_step
1435
and scheduler_outputs.num_lookahead_slots > 0):
1436
# cache the scheduler outputs for the next iteration if we have
1437
# lookahead slots
1438
self._cache_scheduler_outputs_for_multi_step(
1341
- 0, seq_group_metadata_list, scheduler_outputs)
1439
+ 0, seq_group_metadata_list, scheduler_outputs,
1440
+ allow_async_output_proc)
1441
1442
assert seq_group_metadata_list is not None
1443
assert scheduler_outputs is not None
1444
1445
+ assert not (self.scheduler_config.is_multi_step and \
1446
+ allow_async_output_proc)
1447
+
1448
if not scheduler_outputs.is_empty():
1449
finished_requests_ids = self.scheduler[
1450
0].get_and_reset_finished_requests_ids()
@@ -1366,6 +1468,10 @@ class LLMEngine:
1468
# to each of the non-last PP stages for in-place prepare_input.
1469
last_sampled_token_ids=last_sampled_token_ids)
1470
1471
+ if allow_async_output_proc:
1472
+ execute_model_req.output_proc_callback_fn = \
1473
+ self._process_model_outputs
1474
+
1475
output = self.model_executor.execute_model(
1476
execute_model_req=execute_model_req)
1477
@@ -1374,6 +1480,9 @@ class LLMEngine:
1480
if self.scheduler_config.is_multi_step:
1481
self._update_cached_scheduler_output(0, output)
1482
else:
1483
+ if len(self.output_queue) > 0:
1484
+ assert not self.scheduler_config.is_multi_step
1485
+ self._process_model_outputs(is_async=True)
1486
output = []
1487
1488
# Finish the current step for all the sequence groups.
@@ -1382,23 +1491,41 @@ class LLMEngine:
1491
seq_group.finish_step()
1492
1493
if not self._has_remaining_steps(seq_group_metadata_list):
1385
- # clear the cache if we have finished all the steps
1494
+ # clear the cache if we have finished all the steps.
1495
if self.scheduler_config.is_multi_step:
1496
self.cached_scheduler_outputs[0] = SchedulerOutputState()
1388
- request_outputs = self._process_model_outputs(
1389
- output, scheduler_outputs.scheduled_seq_groups,
1390
- scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
1497
1392
- else:
1393
- request_outputs = []
1498
+ # Add results to the output_queue
1499
+ # (for async or non-async postprocessing)
1500
+ self.output_queue.append(
1501
+ (output, seq_group_metadata_list, scheduler_outputs))
1502
1395
- # Log stats.
1396
- self.do_log_stats(scheduler_outputs, output)
1503
+ if output and allow_async_output_proc:
1504
+ assert len(output) == 1, ("Multi step decoding does not work "
1505
+ "with async output processing.")
1506
1398
- # Tracing
1399
- self.do_tracing(scheduler_outputs)
1507
+ self._advance_to_next_step(
1508
+ output[0], seq_group_metadata_list,
1509
+ scheduler_outputs.scheduled_seq_groups)
1510
+
1511
+ if not allow_async_output_proc:
1512
+ self._process_model_outputs(is_async=False)
1513
+
1514
+ # Log stats.
1515
+ self.do_log_stats(scheduler_outputs, output)
1516
+
1517
+ # Tracing
1518
+ self.do_tracing(scheduler_outputs)
1519
+ else:
1520
+ self.request_outputs = []
1521
1522
if not self.has_unfinished_requests():
1523
+ # Drain async postprocessor
1524
+ if len(self.output_queue) > 0:
1525
+ assert not self.scheduler_config.is_multi_step
1526
+ self._process_model_outputs(is_async=True, clear_outputs=False)
1527
+ assert len(self.output_queue) == 0
1528
+
1529
# Stop the execute model loop in parallel workers until there are
1530
# more requests to process. This avoids waiting indefinitely in
1531
# torch.distributed ops which may otherwise timeout, and unblocks
@@ -1406,7 +1533,7 @@ class LLMEngine:
1533
# queued control plane messages, such as add/remove lora adapters.
1534
self.model_executor.stop_remote_worker_execution_loop()
1535
1409
- return request_outputs
1536
+ return self.request_outputs
1537
1538
def _has_remaining_steps(
1539
self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
@@ -1431,12 +1558,14 @@ class LLMEngine:
1558
def _cache_scheduler_outputs_for_multi_step(
1559
self, virtual_engine: int,
1560
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
1434
- scheduler_outputs: SchedulerOutputs) -> None:
1435
- self.cached_scheduler_outputs[
1436
- virtual_engine].seq_group_metadata_list = seq_group_metadata_list
1437
- self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
1438
- scheduler_outputs
1439
- self.cached_scheduler_outputs[virtual_engine].last_output = None
1561
+ scheduler_outputs: SchedulerOutputs,
1562
+ allow_async_output_proc: bool) -> None:
1563
+ co = self.cached_scheduler_outputs[virtual_engine]
1564
+
1565
+ co.seq_group_metadata_list = seq_group_metadata_list
1566
+ co.scheduler_outputs = scheduler_outputs
1567
+ co.allow_async_output_proc = allow_async_output_proc
1568
+ co.last_output = None
1569
1570
def _update_cached_scheduler_output(
1571
self, virtual_engine: int,
@@ -1472,20 +1601,21 @@ class LLMEngine:
1601
raise KeyError(f"Logger with name {logger_name} does not exist.")
1602
del self.stat_loggers[logger_name]
1603
1475
- def do_log_stats(
1476
- self,
1477
- scheduler_outputs: Optional[SchedulerOutputs] = None,
1478
- model_output: Optional[List[SamplerOutput]] = None) -> None:
1604
+ def do_log_stats(self,
1605
+ scheduler_outputs: Optional[SchedulerOutputs] = None,
1606
+ model_output: Optional[List[SamplerOutput]] = None,
1607
+ finished_before: Optional[List[int]] = None) -> None:
1608
"""Forced log when no requests active."""
1609
if self.log_stats:
1481
- stats = self._get_stats(scheduler_outputs, model_output)
1610
+ stats = self._get_stats(scheduler_outputs, model_output,
1611
+ finished_before)
1612
for logger in self.stat_loggers.values():
1613
logger.log(stats)
1614
1485
- def _get_stats(
1486
- self,
1487
- scheduler_outputs: Optional[SchedulerOutputs],
1488
- model_output: Optional[List[SamplerOutput]] = None) -> Stats:
1615
+ def _get_stats(self,
1616
+ scheduler_outputs: Optional[SchedulerOutputs],
1617
+ model_output: Optional[List[SamplerOutput]] = None,
1618
+ finished_before: Optional[List[int]] = None) -> Stats:
1619
"""Get Stats to be Logged to Prometheus.
1620
1621
Args:
@@ -1550,6 +1680,10 @@ class LLMEngine:
1680
# NOTE: This loop assumes prefill seq_groups are before
1681
# decode seq_groups in scheduled_seq_groups.
1682
if scheduler_outputs is not None:
1683
+ # For async postprocessor, already finished sequences need to be
1684
+ # not counted (to avoid double counting)
1685
+ actual_num_batched_tokens = scheduler_outputs.num_batched_tokens # type: ignore
1686
+
1687
num_generation_tokens_from_prefill_groups = 0.
1688
# NOTE: if scheduler_outputs.num_prefill_groups > 0 and
1689
# the len of scheduler_outputs.scheduled_seq_groups is !=
@@ -1558,6 +1692,11 @@ class LLMEngine:
1692
1693
for idx, scheduled_seq_group in enumerate(
1694
scheduler_outputs.scheduled_seq_groups):
1695
+ # Skip double logging when using async output proc
1696
+ if finished_before and idx in finished_before:
1697
+ actual_num_batched_tokens -= 1
1698
+ continue
1699
+
1700
group_was_prefill = idx < scheduler_outputs.num_prefill_groups
1701
seq_group = scheduled_seq_group.seq_group
1702
@@ -1592,7 +1731,6 @@ class LLMEngine:
1731
# Latency timings
1732
time_e2e_requests.append(now -
1733
seq_group.metrics.arrival_time)
1595
-
1734
# Metadata
1735
num_prompt_tokens_requests.append(
1736
len(seq_group.prompt_token_ids))
@@ -1616,7 +1754,7 @@ class LLMEngine:
1754
# + num_generation_tokens_from_prefill_groups (since we generate
1755
# one token on prefills on iters where the prefill finishes).
1756
num_generation_tokens_iter = (
1619
- scheduler_outputs.num_batched_tokens - num_prompt_tokens_iter +
1757
+ actual_num_batched_tokens - num_prompt_tokens_iter +
1758
num_generation_tokens_from_prefill_groups)
1759
1760
# Spec decode, if enabled, emits specialized metrics from the worker in
vllm/engine/output_processor/interfaces.py
+5
-8
@@ -40,13 +40,9 @@ class SequenceGroupOutputProcessor(ABC):
40
# Importing here to avoid cycle.
41
from vllm.engine.output_processor.single_step import (
42
SingleStepOutputProcessor)
43
- return SingleStepOutputProcessor(
44
- scheduler_config,
45
- detokenizer,
46
- scheduler,
47
- seq_counter,
48
- stop_checker,
49
- )
43
+ return SingleStepOutputProcessor(scheduler_config, detokenizer,
44
+ scheduler, seq_counter,
45
+ stop_checker)
46
else:
47
# Importing here to avoid cycle.
48
from vllm.engine.output_processor.multi_step import (
@@ -61,7 +57,8 @@ class SequenceGroupOutputProcessor(ABC):
57
58
@abstractmethod
59
def process_outputs(self, sequence_group: SequenceGroup,
64
- outputs: List[SequenceGroupOutput]) -> None:
60
+ outputs: List[SequenceGroupOutput],
61
+ is_async: bool) -> None:
62
"""Process new token ids for the sequence group. Handles logic such as
63
detokenization, stop checking, and freeing/forking sequences in the
64
scheduler.
vllm/engine/output_processor/multi_step.py
+16
-9
@@ -57,17 +57,28 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
57
"Prompt logprob is not supported by multi step workers. "
58
"(e.g., speculative decode uses multi step workers).")
59
60
- def process_outputs(self, sequence_group: SequenceGroup,
61
- outputs: List[SequenceGroupOutput]) -> None:
60
+ def process_outputs(self,
61
+ sequence_group: SequenceGroup,
62
+ outputs: List[SequenceGroupOutput],
63
+ is_async: bool = False) -> None:
64
"""Append new tokens in the outputs to sequences in the sequence group.
65
66
This only supports sequence groups of size 1. It supports greater than
67
one new token per sequence.
68
67
- This applies logic like stop condition checking and detokenization,
68
- including freeing finished sequences. It also handles cases where there
69
- are tokens emitted after the EOS token.
69
+ This applies logic like stop condition checking and detokenization.
70
+ It also handles cases where there are tokens emitted after
71
+ the EOS token.
72
+
73
+ is_async - Indicates whether this postprocessor runs in
74
+ parallel with the GPU forward pass and is processing
75
+ tokens from the previous step. If this is true, then
76
+ no tokens need to be appended since it is already done
77
+ externally (before the next schedule() call)
78
"""
79
+ # TODO: Add support for async if necessary
80
+ assert not is_async
81
+
82
seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING)
83
84
assert seqs, "expected running sequences"
@@ -138,7 +149,3 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
149
)
150
if seq.is_finished():
151
break
141
-
142
- if seq.is_finished():
143
- for scheduler in self.scheduler:
144
- scheduler.free_seq(seq)
vllm/engine/output_processor/single_step.py
+20
-12
@@ -29,14 +29,9 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
29
that is currently difficult to schedule multiple steps ahead of time.
30
"""
31
32
- def __init__(
33
- self,
34
- scheduler_config: SchedulerConfig,
35
- detokenizer: Detokenizer,
36
- scheduler: List[Scheduler],
37
- seq_counter: Counter,
38
- stop_checker: StopChecker,
39
- ):
32
+ def __init__(self, scheduler_config: SchedulerConfig,
33
+ detokenizer: Detokenizer, scheduler: List[Scheduler],
34
+ seq_counter: Counter, stop_checker: StopChecker):
35
self.scheduler_config = scheduler_config
36
self.detokenizer = detokenizer
37
self.scheduler = scheduler
@@ -44,16 +39,24 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
39
self.stop_checker = stop_checker
40
41
def process_outputs(self, sequence_group: SequenceGroup,
47
- outputs: List[SequenceGroupOutput]) -> None:
42
+ outputs: List[SequenceGroupOutput],
43
+ is_async: bool) -> None:
44
"""Append all new tokens to sequences in the sequence group. Fork any
45
surviving beam candidates; free any unsurviving ones.
46
47
Invokes detokenizer to detokenize new tokens, and also marks sequences
48
as finished if they meet stop conditions.
49
+
50
+ is_async - Indicates whether this postprocessor runs in
51
+ parallel with the GPU forward pass and is processing
52
+ tokens from the previous step. If this is true, then
53
+ no tokens need to be appended since it is already done
54
+ externally (before the next schedule() call)
55
"""
56
assert (len(outputs) == 1
57
), f"{type(self)} does not support multiple outputs per step"
56
- return self._process_sequence_group_outputs(sequence_group, outputs[0])
58
+ return self._process_sequence_group_outputs(sequence_group, outputs[0],
59
+ is_async)
60
61
def process_prompt_logprob(self, seq_group: SequenceGroup,
62
outputs: List[SequenceGroupOutput]) -> None:
@@ -80,14 +83,16 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
83
seq_group.prompt_logprobs.extend(prompt_logprobs)
84
85
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
83
- outputs: SequenceGroupOutput) -> None:
86
+ outputs: SequenceGroupOutput,
87
+ is_async: bool) -> None:
88
sampling_params = seq_group.sampling_params
89
if sampling_params.n == 1 and not sampling_params.use_beam_search:
90
# only have one output sample
91
sample = outputs.samples[0]
92
# only have one sequence
93
seq = seq_group.seqs[0]
90
- seq.append_token_id(sample.output_token, sample.logprobs)
94
+ if not is_async:
95
+ seq.append_token_id(sample.output_token, sample.logprobs)
96
if sampling_params.detokenize and self.detokenizer:
97
new_char_count = self.detokenizer.decode_sequence_inplace(
98
seq, sampling_params)
@@ -104,6 +109,9 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
109
scheduler.free_seq(seq)
110
return
111
112
+ # TODO: Add support for async for beam search
113
+ assert not is_async
114
+
115
# Process samples
116
samples = outputs.samples
117
parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
vllm/entrypoints/llm.py
+10
-1
@@ -129,6 +129,7 @@ class LLM:
129
max_context_len_to_capture: Optional[int] = None,
130
max_seq_len_to_capture: int = 8192,
131
disable_custom_all_reduce: bool = False,
132
+ disable_async_output_proc: bool = False,
133
**kwargs,
134
) -> None:
135
'''
@@ -170,6 +171,7 @@ class LLM:
171
max_context_len_to_capture=max_context_len_to_capture,
172
max_seq_len_to_capture=max_seq_len_to_capture,
173
disable_custom_all_reduce=disable_custom_all_reduce,
174
+ disable_async_output_proc=disable_async_output_proc,
175
**kwargs,
176
)
177
self.llm_engine = LLMEngine.from_engine_args(
@@ -603,7 +605,6 @@ class LLM:
605
inputs = [inputs]
606
607
num_requests = len(inputs)
606
-
608
if isinstance(params, list) and len(params) != num_requests:
609
raise ValueError("The lengths of prompts and params "
610
"must be the same.")
@@ -678,6 +679,10 @@ class LLM:
679
postfix=(f"est. speed input: {0:.2f} toks/s, "
680
f"output: {0:.2f} toks/s"),
681
)
682
+
683
+ # In the loop below, only finished outputs are used
684
+ self.llm_engine.step_return_finished_only = True
685
+
686
# Run the engine.
687
outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
688
total_in_toks = 0
@@ -700,6 +705,10 @@ class LLM:
705
f"est. speed input: {in_spd:.2f} toks/s, "
706
f"output: {out_spd:.2f} toks/s")
707
pbar.update(1)
708
+
709
+ # Restore original behavior
710
+ self.llm_engine.step_return_finished_only = False
711
+
712
if use_tqdm:
713
pbar.close()
714
# Sort the outputs by request ID.
vllm/executor/distributed_gpu_executor.py
+4
-3
@@ -64,8 +64,9 @@ class DistributedGPUExecutor(GPUExecutor):
64
num_cpu_blocks=num_cpu_blocks)
65
66
def execute_model(
67
- self,
68
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
67
+ self,
68
+ execute_model_req: ExecuteModelRequest,
69
+ ) -> List[SamplerOutput]:
70
if self.parallel_worker_tasks is None:
71
self.parallel_worker_tasks = self._run_workers(
72
"start_worker_execution_loop",
@@ -188,7 +189,7 @@ class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
189
@abstractmethod
190
async def _driver_execute_model_async(
191
self,
191
- execute_model_req: Optional[ExecuteModelRequest] = None
192
+ execute_model_req: Optional[ExecuteModelRequest] = None,
193
) -> List[SamplerOutput]:
194
"""Execute the model asynchronously in the driver worker.
195
vllm/executor/gpu_executor.py
+1
-1
@@ -176,5 +176,5 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
176
execute_model_req: ExecuteModelRequest,
177
) -> List[Union[SamplerOutput, PoolerOutput]]:
178
output = await make_async(self.driver_worker.execute_model
179
- )(execute_model_req=execute_model_req, )
179
+ )(execute_model_req=execute_model_req)
180
return output
vllm/sequence.py
+8
-8
@@ -5,8 +5,8 @@ from abc import ABC, abstractmethod
5
from array import array
6
from collections import defaultdict
7
from dataclasses import dataclass
8
-from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
9
- Tuple, Union, cast)
8
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
9
+ Optional, Set, Tuple, Union, cast)
10
11
import msgspec
12
import torch
@@ -474,11 +474,8 @@ class Sequence:
474
"""Reset the sequence states for recomputation."""
475
self.data.reset_state_for_recompute()
476
477
- def append_token_id(
478
- self,
479
- token_id: int,
480
- logprobs: Dict[int, Logprob],
481
- ) -> None:
477
+ def append_token_id(self, token_id: int, logprobs: Dict[int,
478
+ Logprob]) -> None:
479
assert token_id in logprobs
480
self.output_logprobs.append(logprobs)
481
self.data.append_token_id(token_id, logprobs[token_id].logprob)
@@ -1293,6 +1290,8 @@ class ExecuteModelRequest(
1290
finished_requests_ids: List[str] = msgspec.field(default_factory=list)
1291
# The last sampled token ids for multi step decoding.
1292
last_sampled_token_ids: Optional[torch.Tensor] = None
1293
+ # Async postprocessor
1294
+ output_proc_callback_fn: Optional[Callable] = None
1295
1296
@property
1297
def is_first_multi_step(self) -> bool:
@@ -1338,4 +1337,5 @@ class ExecuteModelRequest(
1337
num_steps=self.num_steps,
1338
finished_requests_ids=self.finished_requests_ids,
1339
last_sampled_token_ids=self.last_sampled_token_ids.clone()
1341
- if self.last_sampled_token_ids is not None else None)
1340
+ if self.last_sampled_token_ids is not None else None,
1341
+ output_proc_callback_fn=self.output_proc_callback_fn)
vllm/worker/model_runner.py
+7
-3
@@ -6,8 +6,8 @@ import time
6
import warnings
7
import weakref
8
from dataclasses import dataclass
9
-from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
10
- TypeVar, Union)
9
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
10
+ Tuple, Type, TypeVar, Union)
11
12
import numpy as np
13
import torch
@@ -90,6 +90,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
90
request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
91
finished_requests_ids: Optional[List[str]] = None
92
virtual_engine: int = 0
93
+ output_proc_callback_fn: Optional[Callable] = None
94
95
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
96
tensor_dict = {
@@ -1327,7 +1328,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1328
self,
1329
seq_group_metadata_list: List[SequenceGroupMetadata],
1330
virtual_engine: int = 0,
1330
- finished_requests_ids: Optional[List[str]] = None
1331
+ finished_requests_ids: Optional[List[str]] = None,
1332
) -> ModelInputForGPUWithSamplingMetadata:
1333
"""Prepare the model input based on a given sequence group, including
1334
metadata for the sampling step.
@@ -1451,6 +1452,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1452
if not self.is_driver_worker:
1453
return []
1454
1455
+ if model_input.output_proc_callback_fn is not None:
1456
+ model_input.output_proc_callback_fn(is_async=True)
1457
+
1458
# Sample the next token.
1459
output: SamplerOutput = self.model.sample(
1460
logits=logits,
vllm/worker/worker_base.py
+7
-1
@@ -263,6 +263,12 @@ class LocalOrDistributedWorkerBase(WorkerBase):
263
broadcast_data.update(kwargs)
264
broadcast_tensor_dict(broadcast_data, src=0)
265
266
+ if execute_model_req.output_proc_callback_fn:
267
+ model_input = dataclasses.replace( # type: ignore
268
+ model_input,
269
+ output_proc_callback_fn=execute_model_req.
270
+ output_proc_callback_fn)
271
+
272
return model_input, worker_input, kwargs
273
274
def prepare_input(
@@ -289,7 +295,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
295
296
def execute_model(
297
self,
292
- execute_model_req: Optional[ExecuteModelRequest] = None
298
+ execute_model_req: Optional[ExecuteModelRequest] = None,
299
) -> Optional[List[SamplerOutput]]:
300
"""Executes at least one model step on the given sequences, unless no
301
sequences are provided."""