@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2eedede875
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Asynchronous Output Processor (#7049)

Co-authored-by: Alexander Matveev <alexm@neuralmagic.com>

Megha Agarwal committed Aug 26, 2024 at 20:53 UTC 2eedede87502be64f60962147513f3df6cb1bd01
21 files changed +636 -198
benchmarks/benchmark_throughput.py
+9 -1
@@ -86,6 +86,7 @@ def run_vllm(
86 use_v2_block_manager: bool = False,
87 download_dir: Optional[str] = None,
88 load_format: str = EngineArgs.load_format,
89 + disable_async_output_proc: bool = False,
90 ) -> float:
91 from vllm import LLM, SamplingParams
92 llm = LLM(
@@ -110,6 +111,7 @@ def run_vllm(
111 load_format=load_format,
112 num_scheduler_steps=num_scheduler_steps,
113 use_v2_block_manager=use_v2_block_manager,
114 + disable_async_output_proc=disable_async_output_proc,
115 )
116
117 # Add the requests to the engine.
@@ -237,7 +239,8 @@ def main(args: argparse.Namespace):
239 args.enable_prefix_caching, args.enable_chunked_prefill,
240 args.max_num_batched_tokens, args.distributed_executor_backend,
241 args.gpu_memory_utilization, args.num_scheduler_steps,
240 - args.use_v2_block_manager, args.download_dir, args.load_format)
242 + args.use_v2_block_manager, args.download_dir, args.load_format,
243 + args.disable_async_output_proc)
244 elif args.backend == "hf":
245 assert args.tensor_parallel_size == 1
246 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -418,6 +421,11 @@ if __name__ == "__main__":
421 'section for more information.\n'
422 '* "bitsandbytes" will load the weights using bitsandbytes '
423 'quantization.\n')
424 + parser.add_argument(
425 + "--disable-async-output-proc",
426 + action='store_true',
427 + default=False,
428 + help="Disable async output processor for vLLM backend.")
429 args = parser.parse_args()
430 if args.tokenizer is None:
431 args.tokenizer = args.model
tests/basic_correctness/test_chunked_prefill.py
+6
@@ -88,6 +88,9 @@ def test_models(
88 # NOTE: Increasing this in this suite will fail CI because we currently cannot
89 # reset distributed env properly. Use a value > 1 just when you test.
90 @pytest.mark.parametrize("tensor_parallel_size", [1])
91 +# Due to low-precision numerical divergence, this test is too sensitive to
92 +# the async postprocessor
93 +@pytest.mark.parametrize("disable_async_output_proc", [True])
94 def test_models_with_fp8_kv_cache(
95 vllm_runner,
96 example_prompts,
@@ -97,6 +100,7 @@ def test_models_with_fp8_kv_cache(
100 chunked_prefill_token_size: int,
101 enforce_eager: bool,
102 tensor_parallel_size: int,
103 + disable_async_output_proc: bool,
104 ) -> None:
105 """
106 Only checks log probs match between chunked-prefill and
@@ -126,6 +130,7 @@ def test_models_with_fp8_kv_cache(
130 enforce_eager=enforce_eager,
131 max_num_seqs=max_num_seqs,
132 kv_cache_dtype=kv_cache_dtype,
133 + disable_async_output_proc=disable_async_output_proc,
134 **extra_kwargs,
135 ) as vllm_model:
136 no_chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
@@ -139,6 +144,7 @@ def test_models_with_fp8_kv_cache(
144 enforce_eager=enforce_eager,
145 max_num_seqs=max_num_seqs,
146 kv_cache_dtype=kv_cache_dtype,
147 + disable_async_output_proc=disable_async_output_proc,
148 **extra_kwargs,
149 ) as vllm_model:
150 chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
tests/basic_correctness/test_preemption.py
-1
@@ -209,7 +209,6 @@ def test_swap_infeasible(
209 prefill_blocks = 2
210 decode_blocks = max_tokens // BLOCK_SIZE
211 example_prompts = example_prompts[:1]
212 -
212 with vllm_runner(
213 model,
214 dtype=dtype,
tests/core/test_chunked_prefill_scheduler.py
+2 -2
@@ -21,7 +21,7 @@ def append_new_token(seq_group, token_id: int):
21
22
23 def schedule_and_update_computed_tokens(scheduler):
24 - metas, out = scheduler.schedule()
24 + metas, out, _ = scheduler.schedule()
25 for s, meta in zip(out.scheduled_seq_groups, metas):
26 s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
27 return metas, out
@@ -180,7 +180,7 @@ def test_maximal_decoding():
180 """Verify decoding requests are prioritized."""
181 block_size = 4
182 max_seqs = 2
183 - max_model_len = 2
183 + max_model_len = 8
184 max_num_batched_tokens = 2
185 scheduler_config = SchedulerConfig(max_num_batched_tokens,
186 max_seqs,
tests/core/utils.py
+1 -1
@@ -199,7 +199,7 @@ def append_new_token(out, token_id: int):
199
200
201 def schedule_and_update_computed_tokens(scheduler):
202 - metas, out = scheduler.schedule()
202 + metas, out, _ = scheduler.schedule()
203 for s, meta in zip(out.scheduled_seq_groups, metas):
204 s.seq_group.update_num_computed_tokens(meta.token_chunk_size)
205 return metas, out
tests/engine/test_stop_strings.py
+103 -52
@@ -7,6 +7,8 @@ from vllm import CompletionOutput, LLMEngine, SamplingParams
7 MODEL = "meta-llama/llama-2-7b-hf"
8 MAX_TOKENS = 200
9
10 +IS_ASYNC = False
11 +
12
13 @pytest.fixture(scope="session")
14 def vllm_model(vllm_runner):
@@ -14,99 +16,148 @@ def vllm_model(vllm_runner):
16 yield vllm_model
17
18
17 -@pytest.mark.skip_global_cleanup
18 -def test_stop_basic(vllm_model):
19 - _test_stopping(vllm_model.model.llm_engine,
19 +def _test_stopping(llm_engine: LLMEngine,
20 + expected_output: str,
21 + expected_reason: Any,
22 + stop: Optional[List[str]] = None,
23 + stop_token_ids: Optional[List[int]] = None,
24 + include_in_output: bool = False,
25 + use_async_output_proc: bool = False) -> None:
26 + llm_engine.add_request(
27 + "id", "A story about vLLM:\n",
28 + SamplingParams(
29 + temperature=0.0,
30 + max_tokens=MAX_TOKENS,
31 + stop=stop,
32 + stop_token_ids=stop_token_ids,
33 + include_stop_str_in_output=include_in_output,
34 + ), None)
35 +
36 + output: Optional[CompletionOutput] = None
37 + output_text = ""
38 + stop_reason = None
39 +
40 + if use_async_output_proc:
41 + llm_engine.step()
42 +
43 + while llm_engine.has_unfinished_requests():
44 + (request_output, ) = llm_engine.step()
45 + (output, ) = request_output.outputs
46 +
47 + # Ensure we don't backtrack
48 + assert output.text.startswith(output_text)
49 + output_text = output.text
50 + stop_reason = output.stop_reason
51 +
52 + assert output is not None
53 + assert output_text == expected_output
54 + assert stop_reason == expected_reason
55 +
56 +
57 +def _set_async_mode(llm_engine, is_async):
58 + llm_engine.scheduler[0].use_async_output_proc = is_async
59 +
60 +
61 +def _stop_basic(llm_engine, is_async):
62 + _test_stopping(llm_engine,
63 stop=["."],
64 include_in_output=False,
65 expected_output="VLLM is a 100% volunteer organization",
23 - expected_reason=".")
66 + expected_reason=".",
67 + use_async_output_proc=is_async)
68
25 - _test_stopping(vllm_model.model.llm_engine,
69 + _test_stopping(llm_engine,
70 stop=["."],
71 include_in_output=True,
72 expected_output="VLLM is a 100% volunteer organization.",
29 - expected_reason=".")
73 + expected_reason=".",
74 + use_async_output_proc=is_async)
75
76
32 -@pytest.mark.skip_global_cleanup
33 -def test_stop_multi_tokens(vllm_model):
77 +def _stop_multi_tokens(llm_engine, is_async):
78 _test_stopping(
35 - vllm_model.model.llm_engine,
79 + llm_engine,
80 stop=["group of peo", "short"],
81 include_in_output=False,
82 expected_output="VLLM is a 100% volunteer organization. We are a ",
39 - expected_reason="group of peo")
83 + expected_reason="group of peo",
84 + use_async_output_proc=is_async)
85
86 _test_stopping(
42 - vllm_model.model.llm_engine,
87 + llm_engine,
88 stop=["group of peo", "short"],
89 include_in_output=True,
90 expected_output=
91 "VLLM is a 100% volunteer organization. We are a group of peo",
47 - expected_reason="group of peo")
92 + expected_reason="group of peo",
93 + use_async_output_proc=is_async)
94
95
50 -@pytest.mark.skip_global_cleanup
51 -def test_stop_partial_token(vllm_model):
52 - _test_stopping(vllm_model.model.llm_engine,
96 +def _stop_partial_token(llm_engine, is_async):
97 + _test_stopping(llm_engine,
98 stop=["gani"],
99 include_in_output=False,
100 expected_output="VLLM is a 100% volunteer or",
56 - expected_reason="gani")
101 + expected_reason="gani",
102 + use_async_output_proc=is_async)
103
58 - _test_stopping(vllm_model.model.llm_engine,
104 + _test_stopping(llm_engine,
105 stop=["gani"],
106 include_in_output=True,
107 expected_output="VLLM is a 100% volunteer organi",
62 - expected_reason="gani")
108 + expected_reason="gani",
109 + use_async_output_proc=is_async)
110
111
65 -@pytest.mark.skip_global_cleanup
66 -def test_stop_token_id(vllm_model):
112 +def _stop_token_id(llm_engine, is_async):
113 # token id 13013 => " organization"
114
69 - _test_stopping(vllm_model.model.llm_engine,
115 + _test_stopping(llm_engine,
116 stop_token_ids=[13013],
117 include_in_output=False,
118 expected_output="VLLM is a 100% volunteer",
73 - expected_reason=13013)
119 + expected_reason=13013,
120 + use_async_output_proc=is_async)
121
75 - _test_stopping(vllm_model.model.llm_engine,
122 + _test_stopping(llm_engine,
123 stop_token_ids=[13013],
124 include_in_output=True,
125 expected_output="VLLM is a 100% volunteer organization",
79 - expected_reason=13013)
126 + expected_reason=13013,
127 + use_async_output_proc=is_async)
128
129
82 -def _test_stopping(llm_engine: LLMEngine,
83 - expected_output: str,
84 - expected_reason: Any,
85 - stop: Optional[List[str]] = None,
86 - stop_token_ids: Optional[List[int]] = None,
87 - include_in_output: bool = False) -> None:
88 - llm_engine.add_request(
89 - "id", "A story about vLLM:\n",
90 - SamplingParams(
91 - temperature=0.0,
92 - max_tokens=MAX_TOKENS,
93 - stop=stop,
94 - stop_token_ids=stop_token_ids,
95 - include_stop_str_in_output=include_in_output,
96 - ), None)
130 +@pytest.mark.skip_global_cleanup
131 +def test_stop_basic(vllm_model):
132 + _set_async_mode(vllm_model.model.llm_engine, True)
133 + _stop_basic(vllm_model.model.llm_engine, is_async=True)
134
98 - output: Optional[CompletionOutput] = None
99 - output_text = ""
100 - stop_reason = None
101 - while llm_engine.has_unfinished_requests():
102 - (request_output, ) = llm_engine.step()
103 - (output, ) = request_output.outputs
135 + _set_async_mode(vllm_model.model.llm_engine, False)
136 + _stop_basic(vllm_model.model.llm_engine, is_async=False)
137
105 - # Ensure we don't backtrack
106 - assert output.text.startswith(output_text)
107 - output_text = output.text
108 - stop_reason = output.stop_reason
138
110 - assert output is not None
111 - assert output_text == expected_output
112 - assert stop_reason == expected_reason
139 +@pytest.mark.skip_global_cleanup
140 +def test_stop_multi_tokens(vllm_model):
141 + _set_async_mode(vllm_model.model.llm_engine, True)
142 + _stop_multi_tokens(vllm_model.model.llm_engine, is_async=True)
143 +
144 + _set_async_mode(vllm_model.model.llm_engine, False)
145 + _stop_multi_tokens(vllm_model.model.llm_engine, is_async=False)
146 +
147 +
148 +@pytest.mark.skip_global_cleanup
149 +def test_stop_partial_token(vllm_model):
150 + _set_async_mode(vllm_model.model.llm_engine, True)
151 + _stop_partial_token(vllm_model.model.llm_engine, is_async=True)
152 +
153 + _set_async_mode(vllm_model.model.llm_engine, False)
154 + _stop_partial_token(vllm_model.model.llm_engine, is_async=False)
155 +
156 +
157 +@pytest.mark.skip_global_cleanup
158 +def test_stop_token_id(vllm_model):
159 + _set_async_mode(vllm_model.model.llm_engine, True)
160 + _stop_token_id(vllm_model.model.llm_engine, is_async=True)
161 +
162 + _set_async_mode(vllm_model.model.llm_engine, False)
163 + _stop_token_id(vllm_model.model.llm_engine, is_async=False)
tests/multi_step/test_correctness_async_llm.py
+3
@@ -62,6 +62,9 @@ async def test_multi_step(example_prompts, model: str, tp_size: int,
62 ms_server_args = DEFAULT_SERVER_ARGS + \
63 ["--num-scheduler-steps", f"{num_scheduler_steps}"]
64
65 + # Disable output proc callback as its not supported
66 + # with multi-step right now
67 + ms_server_args += ["--disable-async-output-proc"]
68 if eager_mode:
69 ms_server_args.append("--enforce-eager")
70
vllm/config.py
+53
@@ -140,6 +140,7 @@ class ModelConfig:
140 skip_tokenizer_init: bool = False,
141 served_model_name: Optional[Union[str, List[str]]] = None,
142 limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
143 + use_async_output_proc: bool = True,
144 ) -> None:
145 self.model = model
146 self.tokenizer = tokenizer
@@ -172,6 +173,7 @@ class ModelConfig:
173 self.hf_image_processor_config = get_hf_image_processor_config(
174 self.model, revision)
175 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
176 + self.use_async_output_proc = use_async_output_proc
177
178 # Choose a default enforce_eager value if the user did not specify
179 # a value (enforce_eager is None)
@@ -326,6 +328,49 @@ class ModelConfig:
328 self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
329 self.max_model_len)
330
331 + def verify_async_output_proc(self, parallel_config, speculative_config,
332 + device_config) -> None:
333 + if not self.use_async_output_proc:
334 + # Nothing to check
335 + return
336 +
337 + if parallel_config.pipeline_parallel_size > 1:
338 + logger.warning("Async output processing can not be enabled "
339 + "with pipeline parallel")
340 + self.use_async_output_proc = False
341 + return
342 +
343 + if device_config.device_type != "cuda":
344 + logger.warning(
345 + "Async output processing is only supported for CUDA."
346 + " Disabling it for other platforms.")
347 + self.use_async_output_proc = False
348 + return
349 +
350 + if envs.VLLM_USE_RAY_SPMD_WORKER:
351 + logger.warning(
352 + "Async output processing can not be enabled with ray spmd")
353 + self.use_async_output_proc = False
354 + return
355 +
356 + if self.enforce_eager:
357 + logger.warning(
358 + "To see benefits of async output processing, enable CUDA "
359 + "graph. Since, enforce-eager is enabled, async output "
360 + "processor cannot be used")
361 + self.use_async_output_proc = not self.enforce_eager
362 + return
363 +
364 + # Async postprocessor is not necessary with embedding mode
365 + # since there is no token generation
366 + if self.embedding_mode:
367 + self.use_async_output_proc = False
368 +
369 + if speculative_config:
370 + logger.warning("Async output processing is not supported with"
371 + " speculative decoding currently.")
372 + self.use_async_output_proc = False
373 +
374 def verify_with_parallel_config(
375 self,
376 parallel_config: "ParallelConfig",
@@ -358,6 +403,11 @@ class ModelConfig:
403 "fallback to the eager mode.")
404 self.enforce_eager = True
405
406 + if pipeline_parallel_size > 1 and self.use_async_output_proc:
407 + logger.warning("Async output processor is not supported with "
408 + "pipeline parallelism currently. Disabling it.")
409 + self.use_async_output_proc = False
410 +
411 def get_hf_config_sliding_window(self) -> Optional[int]:
412 """Get the sliding window size, or None if disabled."""
413
@@ -1769,6 +1819,9 @@ class EngineConfig:
1819 def __post_init__(self):
1820 """Verify configs are valid & consistent with each other.
1821 """
1822 + self.model_config.verify_async_output_proc(self.parallel_config,
1823 + self.speculative_config,
1824 + self.device_config)
1825 self.model_config.verify_with_parallel_config(self.parallel_config)
1826 self.cache_config.verify_with_parallel_config(self.parallel_config)
1827
vllm/core/scheduler.py
+118 -12
@@ -4,7 +4,8 @@ import random
4 import time
5 from collections import deque
6 from dataclasses import dataclass, field
7 -from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
7 +from typing import (Callable, Deque, Dict, Iterable, List, Optional, Set,
8 + Tuple, Union)
9
10 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
11 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
@@ -299,6 +300,7 @@ class Scheduler:
300 cache_config: CacheConfig,
301 lora_config: Optional[LoRAConfig],
302 pipeline_parallel_size: int = 1,
303 + output_proc_callback_fn: Optional[Callable] = None,
304 ) -> None:
305 self.scheduler_config = scheduler_config
306 self.cache_config = cache_config
@@ -364,10 +366,36 @@ class Scheduler:
366 self.num_cumulative_preemption: int = 0
367
368 # Used to cache python objects
367 - self._scheduler_running_outputs_cache: PyObjectCache = PyObjectCache(
368 - scheduler_running_outputs_builder)
369 - self._scheduled_seq_group_cache: PyObjectCache = PyObjectCache(
370 - scheduled_seq_group_builder)
369 + self._seq_group_metadata_cache: List[PyObjectCache] = []
370 + self._scheduler_running_outputs_cache: List[PyObjectCache] = []
371 + self._scheduled_seq_group_cache: List[PyObjectCache] = []
372 +
373 + # For async output processing, we need to swap cache buffers between
374 + # iterations. I.e. since the output processing is lagged one step,
375 + # we cannot reuse the cached objects immediately when the schedule()
376 + # is called again, but only when schedule() is called the second time.
377 + self.output_proc_callback_fn = output_proc_callback_fn
378 + self.use_async_output_proc = self.output_proc_callback_fn is not None
379 + self.num_cache_iters = 2 if self.use_async_output_proc else 1
380 +
381 + self.cache_id = 0
382 + for i in range(self.num_cache_iters):
383 + self._seq_group_metadata_cache.append(
384 + PyObjectCache(seq_group_metadata_builder))
385 + self._scheduler_running_outputs_cache.append(
386 + PyObjectCache(scheduler_running_outputs_builder))
387 + self._scheduled_seq_group_cache.append(
388 + PyObjectCache(scheduled_seq_group_builder))
389 +
390 + # For async postprocessor, the extra decode run cannot be done
391 + # when the request reaches max_model_len. In this case, the request
392 + # will be stopped during schedule() call and added to this stop list
393 + # for processing and deallocation by the free_finished_seq_groups()
394 + self._async_stopped: List[SequenceGroup] = []
395 +
396 + @property
397 + def next_cache_id(self):
398 + return (self.cache_id + 1) % self.num_cache_iters
399
400 @property
401 def lora_enabled(self) -> bool:
@@ -483,7 +511,7 @@ class Scheduler:
511 SchedulerRunningOutputs.
512 """
513 ret: SchedulerRunningOutputs = \
486 - self._scheduler_running_outputs_cache.get_object()
514 + self._scheduler_running_outputs_cache[self.cache_id].get_object()
515 ret.blocks_to_swap_out.clear()
516 ret.blocks_to_copy.clear()
517 ret.decode_seq_groups.clear()
@@ -510,8 +538,12 @@ class Scheduler:
538 # NOTE(woosuk): Preemption happens only when there is no available slot
539 # to keep all the sequence groups in the RUNNING state.
540
513 - running_queue = self.running
541 + # Store original running requests for the case of async + preemption
542 + if self.use_async_output_proc:
543 + orig_running = self.running.copy()
544
545 + running_queue = self.running
546 + assert len(self._async_stopped) == 0
547 while running_queue:
548 seq_group = running_queue[0]
549 num_running_tokens = self._get_num_new_tokens(
@@ -521,6 +553,28 @@ class Scheduler:
553 break
554
555 running_queue.popleft()
556 +
557 + # With async postprocessor, an extra decode run is done
558 + # to process the final tokens. The check below avoids this extra
559 + # decode run when the model max len is reached, in order to avoid
560 + # a memory overflow.
561 + if self.use_async_output_proc and seq_group.seqs[0].get_len(
562 + ) > self.scheduler_config.max_model_len:
563 + self._async_stopped.append(seq_group)
564 + continue
565 +
566 + # With async postprocessor, when preemption kicks in, we need
567 + # first to drain the async postprocessor, so that all async
568 + # block_table freeing is applied before the preemption freeing
569 + # is applied.
570 + if self.use_async_output_proc and not self._can_append_slots(
571 + seq_group):
572 + tmp = self.running
573 + self.running = orig_running
574 + assert self.output_proc_callback_fn is not None
575 + self.output_proc_callback_fn(is_async=True)
576 + self.running = tmp
577 +
578 while not self._can_append_slots(seq_group):
579 budget.subtract_num_batched_tokens(seq_group.request_id,
580 num_running_tokens)
@@ -556,7 +610,7 @@ class Scheduler:
610 is_prefill = seq_group.is_prefill()
611
612 scheduled_seq_group: ScheduledSequenceGroup = \
559 - self._scheduled_seq_group_cache.get_object()
613 + self._scheduled_seq_group_cache[self.cache_id].get_object()
614 scheduled_seq_group.seq_group = seq_group
615 if is_prefill:
616 scheduled_seq_group.token_chunk_size = num_running_tokens
@@ -579,8 +633,8 @@ class Scheduler:
633 if curr_loras is not None and seq_group.lora_int_id > 0:
634 curr_loras.add(seq_group.lora_int_id)
635
582 - self._scheduler_running_outputs_cache.reset()
583 - self._scheduled_seq_group_cache.reset()
636 + self._scheduler_running_outputs_cache[self.next_cache_id].reset()
637 + self._scheduled_seq_group_cache[self.next_cache_id].reset()
638
639 return ret
640
@@ -1031,17 +1085,31 @@ class Scheduler:
1085 num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
1086 )
1087
1034 - def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
1088 + def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1089 + no_beam_search = (seq_group.sampling_params.best_of == 1
1090 + and not seq_group.sampling_params.use_beam_search)
1091 +
1092 + return no_beam_search
1093 +
1094 + def schedule(
1095 + self
1096 + ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, bool]:
1097 # Schedule sequence groups.
1098 # This function call changes the internal states of the scheduler
1099 # such as self.running, self.swapped, and self.waiting.
1100 scheduler_start_time = time.perf_counter()
1101 +
1102 scheduler_outputs = self._schedule()
1103 now = time.time()
1104
1105 if not self.cache_config.enable_prefix_caching:
1106 common_computed_block_nums = []
1107
1108 + # TODO: Combine multi-step and async postprocessor
1109 + allow_async_output_proc: bool = (
1110 + self.use_async_output_proc
1111 + and not self.scheduler_config.is_multi_step)
1112 +
1113 # Create input data structures.
1114 seq_group_metadata_list: List[SequenceGroupMetadata] = []
1115 for i, scheduled_seq_group in enumerate(
@@ -1050,6 +1118,11 @@ class Scheduler:
1118 token_chunk_size = scheduled_seq_group.token_chunk_size
1119 seq_group.maybe_set_first_scheduled_time(now)
1120
1121 + seq_group_metadata = self._seq_group_metadata_cache[
1122 + self.cache_id].get_object()
1123 + seq_group_metadata.seq_data.clear()
1124 + seq_group_metadata.block_tables.clear()
1125 +
1126 # seq_id -> SequenceData
1127 seq_data: Dict[int, SequenceData] = {}
1128 # seq_id -> physical block numbers
@@ -1139,6 +1212,10 @@ class Scheduler:
1212 )
1213 seq_group_metadata_list.append(seq_group_metadata)
1214
1215 + if allow_async_output_proc:
1216 + allow_async_output_proc = self._allow_async_output_proc(
1217 + seq_group)
1218 +
1219 # Now that the batch has been created, we can assume all blocks in the
1220 # batch will have been computed before the next scheduling invocation.
1221 # This is because the engine assumes that a failure in model execution
@@ -1147,6 +1224,8 @@ class Scheduler:
1224 self.block_manager.mark_blocks_as_computed(
1225 scheduled_seq_group.seq_group)
1226
1227 + self._seq_group_metadata_cache[self.next_cache_id].reset()
1228 +
1229 scheduler_time = time.perf_counter() - scheduler_start_time
1230 # Add this to scheduler time to all the sequences that are currently
1231 # running. This will help estimate if the scheduler is a significant
@@ -1158,7 +1237,12 @@ class Scheduler:
1237 else:
1238 seq_group.metrics.scheduler_time = scheduler_time
1239
1161 - return seq_group_metadata_list, scheduler_outputs
1240 + # Move to next cache (if exists)
1241 + self.cache_id = self.next_cache_id
1242 +
1243 + # Return results
1244 + return (seq_group_metadata_list, scheduler_outputs,
1245 + allow_async_output_proc)
1246
1247 def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
1248 self.block_manager.fork(parent_seq, child_seq)
@@ -1167,6 +1251,12 @@ class Scheduler:
1251 """Free a sequence from a block table."""
1252 self.block_manager.free(seq)
1253
1254 + def _free_finished_seqs(self, seq_group: SequenceGroup) -> None:
1255 + """Free finished seqs in a sequence group."""
1256 + for seq in seq_group.get_seqs():
1257 + if seq.is_finished():
1258 + self.free_seq(seq)
1259 +
1260 def free_finished_seq_groups(self) -> None:
1261 remaining: Deque[SequenceGroup] = deque()
1262 for seq_group in self.running:
@@ -1179,8 +1269,24 @@ class Scheduler:
1269 self._finished_requests_ids.append(seq_group.request_id)
1270 else:
1271 remaining.append(seq_group)
1272 +
1273 + # Free finished seqs
1274 + self._free_finished_seqs(seq_group)
1275 +
1276 self.running = remaining
1277
1278 + # Handle async stopped sequence groups
1279 + # (ones that reached max model len)
1280 + if self._async_stopped:
1281 + for seq_group in self._async_stopped:
1282 + self._free_seq_group_cross_attn_blocks(seq_group)
1283 + self._finished_requests_ids.append(seq_group.request_id)
1284 +
1285 + # Free finished seqs
1286 + self._free_finished_seqs(seq_group)
1287 +
1288 + self._async_stopped.clear()
1289 +
1290 def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
1291 self.block_manager.allocate(seq_group)
1292 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
vllm/engine/arg_utils.py
+8
@@ -147,6 +147,7 @@ class EngineArgs:
147
148 otlp_traces_endpoint: Optional[str] = None
149 collect_detailed_traces: Optional[str] = None
150 + disable_async_output_proc: bool = False
151
152 def __post_init__(self):
153 if self.tokenizer is None:
@@ -733,6 +734,12 @@ class EngineArgs:
734 "modules. This involves use of possibly costly and or blocking "
735 "operations and hence might have a performance impact.")
736
737 + parser.add_argument(
738 + '--disable-async-output-proc',
739 + action='store_true',
740 + default=EngineArgs.disable_async_output_proc,
741 + help="Disable async output processing. This may result in "
742 + "lower performance.")
743 return parser
744
745 @classmethod
@@ -792,6 +799,7 @@ class EngineArgs:
799 skip_tokenizer_init=self.skip_tokenizer_init,
800 served_model_name=self.served_model_name,
801 limit_mm_per_prompt=self.limit_mm_per_prompt,
802 + use_async_output_proc=not self.disable_async_output_proc,
803 )
804 cache_config = CacheConfig(
805 block_size=self.block_size if self.device != "neuron" else
vllm/engine/async_llm_engine.py
+47 -13
@@ -277,23 +277,36 @@ class _AsyncLLMEngine(LLMEngine):
277 cached_outputs = self.cached_scheduler_outputs[virtual_engine]
278 seq_group_metadata_list = cached_outputs.seq_group_metadata_list
279 scheduler_outputs = cached_outputs.scheduler_outputs
280 + allow_async_output_proc = cached_outputs.allow_async_output_proc
281 +
282 # skip the scheduler if there are any remaining steps in the seq groups.
283 # This ensures that the scheduler is only called again when the current
284 # batch has completed.
285 if not self._has_remaining_steps(seq_group_metadata_list):
284 - seq_group_metadata_list, scheduler_outputs = self.scheduler[
285 - virtual_engine].schedule()
286 + (seq_group_metadata_list, scheduler_outputs,
287 + allow_async_output_proc
288 + ) = self.scheduler[virtual_engine].schedule()
289 +
290 + # If current scheduler iteration has no async postprocessor,
291 + # then we need first to drain the pending async postprocessor
292 + # before moving forward
293 + if not allow_async_output_proc and len(self.output_queue) > 0:
294 + self._process_model_outputs(is_async=True)
295
296 if (self.scheduler_config.is_multi_step
297 and scheduler_outputs.num_lookahead_slots > 0):
298 # cache the scheduler outputs for the next iteration if we have
299 # lookahead slots
300 self._cache_scheduler_outputs_for_multi_step(
292 - virtual_engine, seq_group_metadata_list, scheduler_outputs)
301 + virtual_engine, seq_group_metadata_list, scheduler_outputs,
302 + allow_async_output_proc)
303
304 assert seq_group_metadata_list is not None
305 assert scheduler_outputs is not None
306
307 + assert not (self.scheduler_config.is_multi_step and \
308 + allow_async_output_proc)
309 +
310 if not scheduler_outputs.is_empty():
311 finished_requests_ids = self.scheduler[
312 virtual_engine].get_and_reset_finished_requests_ids()
@@ -317,6 +330,11 @@ class _AsyncLLMEngine(LLMEngine):
330 # We use ExecuteModelRequest to pass the last sampled_token_ids
331 # to each of the non-last PP stages for in-place prepare_input.
332 last_sampled_token_ids=last_sampled_token_ids)
333 +
334 + if allow_async_output_proc:
335 + execute_model_req.output_proc_callback_fn = \
336 + self._process_model_outputs
337 +
338 # Execute the model.
339 output = await self.model_executor.execute_model_async(
340 execute_model_req)
@@ -325,6 +343,9 @@ class _AsyncLLMEngine(LLMEngine):
343 if self.scheduler_config.is_multi_step:
344 self._update_cached_scheduler_output(virtual_engine, output)
345 else:
346 + if len(self.output_queue) > 0:
347 + assert not self.scheduler_config.is_multi_step
348 + self._process_model_outputs(is_async=True)
349 output = []
350
351 # Finish the current step for all the sequence groups.
@@ -337,19 +358,32 @@ class _AsyncLLMEngine(LLMEngine):
358 if self.scheduler_config.is_multi_step:
359 self.cached_scheduler_outputs[
360 virtual_engine] = SchedulerOutputState()
340 - request_outputs = self._process_model_outputs(
341 - output, scheduler_outputs.scheduled_seq_groups,
342 - scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
343 - else:
344 - request_outputs = []
361
346 - # Log stats.
347 - self.do_log_stats(scheduler_outputs, output)
362 + # Cache results in engine
363 + self.output_queue.append(
364 + (output, seq_group_metadata_list, scheduler_outputs))
365
349 - # Tracing
350 - self.do_tracing(scheduler_outputs)
366 + if output and allow_async_output_proc:
367 + assert len(
368 + output
369 + ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
370 + self._advance_to_next_step(
371 + output[0], seq_group_metadata_list,
372 + scheduler_outputs.scheduled_seq_groups)
373 +
374 + if not allow_async_output_proc:
375 + self._process_model_outputs(is_async=False)
376 +
377 + # Log stats.
378 + self.do_log_stats(scheduler_outputs, output)
379 +
380 + # Tracing
381 + self.do_tracing(scheduler_outputs)
382 +
383 + else:
384 + self.request_outputs = []
385
352 - return request_outputs
386 + return self.request_outputs
387
388 async def stop_remote_worker_execution_loop_async(self) -> None:
389 """Stop the remote worker execution loop."""
vllm/engine/llm_engine.py
+208 -70
@@ -1,7 +1,8 @@
1 import time
2 +from collections import deque
3 from contextlib import contextmanager
4 from dataclasses import dataclass
4 -from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5 +from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
6 Mapping, Optional)
7 from typing import Sequence as GenericSequence
8 from typing import Set, Tuple, Type, Union
@@ -38,9 +39,8 @@ from vllm.pooling_params import PoolingParams
39 from vllm.prompt_adapter.request import PromptAdapterRequest
40 from vllm.sampling_params import SamplingParams
41 from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
41 - PoolerOutput, SamplerOutput, Sequence,
42 - SequenceGroup, SequenceGroupMetadata,
43 - SequenceStatus)
42 + SamplerOutput, Sequence, SequenceGroup,
43 + SequenceGroupMetadata, SequenceStatus)
44 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
45 init_tracer)
46 from vllm.transformers_utils.config import try_get_generation_config
@@ -82,9 +82,10 @@ DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
82 @dataclass
83 class SchedulerOutputState:
84 """Caches the scheduler outputs for a virtual engine. Used for Multi-Step"""
85 - last_output: Optional[SamplerOutput] = None
85 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
86 scheduler_outputs: Optional[SchedulerOutputs] = None
87 + allow_async_output_proc: bool = False
88 + last_output: Optional[SamplerOutput] = None
89
90
91 class LLMEngine:
@@ -190,6 +191,9 @@ class LLMEngine:
191 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
192 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
193 input_registry: InputRegistry = INPUT_REGISTRY,
194 + # To improve performance, only final requests outputs may be required.
195 + # If this set to true, then no intermediate outputs will be returned.
196 + step_return_finished_only: bool = False,
197 ) -> None:
198 logger.info(
199 "Initializing an LLM engine (v%s) with config: "
@@ -204,7 +208,8 @@ class LLMEngine:
208 "quantization_param_path=%s, device_config=%s, "
209 "decoding_config=%r, observability_config=%r, "
210 "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
207 - "num_scheduler_steps=%d, enable_prefix_caching=%s)",
211 + "num_scheduler_steps=%d, enable_prefix_caching=%s, "
212 + "use_async_output_proc=%s)",
213 VLLM_VERSION,
214 model_config.model,
215 speculative_config,
@@ -235,6 +240,7 @@ class LLMEngine:
240 scheduler_config.use_v2_block_manager,
241 scheduler_config.num_scheduler_steps,
242 cache_config.enable_prefix_caching,
243 + model_config.use_async_output_proc,
244 )
245 # TODO(woosuk): Print more configs in debug mode.
246 from vllm.plugins import load_general_plugins
@@ -253,6 +259,7 @@ class LLMEngine:
259 self.observability_config = observability_config or ObservabilityConfig(
260 )
261 self.log_stats = log_stats
262 + self.step_return_finished_only = step_return_finished_only
263
264 if not self.model_config.skip_tokenizer_init:
265 self.tokenizer = self._init_tokenizer()
@@ -340,8 +347,11 @@ class LLMEngine:
347 # NOTE: the cache_config here have been updated with the numbers of
348 # GPU and CPU blocks, which are profiled in the distributed executor.
349 self.scheduler = [
343 - Scheduler(scheduler_config, cache_config, lora_config,
344 - parallel_config.pipeline_parallel_size)
350 + Scheduler(
351 + scheduler_config, cache_config, lora_config,
352 + parallel_config.pipeline_parallel_size,
353 + self._process_model_outputs
354 + if model_config.use_async_output_proc else None)
355 for _ in range(parallel_config.pipeline_parallel_size)
356 ]
357
@@ -396,6 +406,13 @@ class LLMEngine:
406 for _ in range(self.parallel_config.pipeline_parallel_size)
407 ]
408
409 + # Async output processing pointers
410 + self.output_queue: Deque[Tuple[List[SamplerOutput],
411 + List[SequenceGroupMetadata],
412 + SchedulerOutputs]] = deque()
413 + self.request_outputs: List[Union[RequestOutput,
414 + EmbeddingRequestOutput]] = []
415 +
416 def _initialize_kv_caches(self) -> None:
417 """Initialize the KV cache in the worker(s).
418
@@ -1197,34 +1214,66 @@ class LLMEngine:
1214
1215 return
1216
1200 - def _process_model_outputs(
1201 - self,
1202 - output: GenericSequence[Union[SamplerOutput, PoolerOutput]],
1203 - scheduled_seq_groups: List[ScheduledSequenceGroup],
1204 - ignored_seq_groups: List[SequenceGroup],
1205 - seq_group_metadata_list: List[SequenceGroupMetadata],
1206 - ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1217 + def _process_model_outputs(self,
1218 + is_async: bool,
1219 + clear_outputs: bool = True) -> None:
1220 """Apply the model output to the sequences in the scheduled seq groups.
1221
1222 + is_async: Indicates whether this postprocessor runs in
1223 + parallel with the GPU forward pass and is processing
1224 + tokens from the previous step. If this is true, then
1225 + no tokens need to be appended since it is already done
1226 + externally (before the next schedule() call)
1227 + clear_outputs: Sometimes existing outputs need to be combined
1228 + with outputs of this call. This happens for postprocessor
1229 + draining at the final stage (like when sequences are finished)
1230 +
1231 Returns RequestOutputs that can be returned to the client.
1232 """
1211 -
1233 now = time.time()
1234
1214 - # Organize outputs by [sequence group][step] instead of
1215 - # [step][sequence group].
1216 - output_by_sequence_group = create_output_by_sequence_group(
1217 - output, num_seq_groups=len(scheduled_seq_groups))
1235 + if clear_outputs:
1236 + self.request_outputs.clear()
1237 +
1238 + if len(self.output_queue) == 0:
1239 + return None
1240 +
1241 + (outputs, seq_group_metadata_list,
1242 + scheduler_outputs) = self.output_queue.popleft()
1243 +
1244 + # Sanity check
1245 + assert len(seq_group_metadata_list) == len(
1246 + scheduler_outputs.scheduled_seq_groups)
1247 +
1248 + # Organize outputs by [step][sequence group] instead of
1249 + # [sequence group][step].
1250 + if len(outputs) > 1:
1251 + outputs_by_sequence_group = create_output_by_sequence_group(
1252 + outputs, num_seq_groups=len(seq_group_metadata_list))
1253 + else:
1254 + outputs_by_sequence_group = outputs
1255 +
1256 + finished_before: List[int] = []
1257 + for i, seq_group_meta in enumerate(seq_group_metadata_list):
1258 + scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1259
1219 - # Update the scheduled sequence groups with the model outputs.
1220 - for scheduled_seq_group, outputs, seq_group_meta in zip(
1221 - scheduled_seq_groups, output_by_sequence_group,
1222 - seq_group_metadata_list):
1260 seq_group = scheduled_seq_group.seq_group
1224 - seq_group.update_num_computed_tokens(
1225 - scheduled_seq_group.token_chunk_size)
1226 - if output is not None and len(output) > 0:
1227 - for o in output:
1261 +
1262 + if seq_group.is_finished():
1263 + finished_before.append(i)
1264 + continue
1265 +
1266 + if len(outputs) > 1:
1267 + output = outputs_by_sequence_group[i]
1268 + else:
1269 + output = [outputs_by_sequence_group[0][i]]
1270 +
1271 + if not is_async:
1272 + seq_group.update_num_computed_tokens(
1273 + scheduled_seq_group.token_chunk_size)
1274 +
1275 + if outputs:
1276 + for o in outputs:
1277 if (isinstance(o, SamplerOutput)
1278 and seq_group.metrics is not None):
1279 if seq_group.metrics.model_forward_time is not None:
@@ -1239,30 +1288,75 @@ class LLMEngine:
1288 else:
1289 seq_group.metrics.model_execute_time = (
1290 o.model_execute_time)
1291 +
1292 if self.model_config.embedding_mode:
1243 - self._process_sequence_group_outputs(seq_group, outputs)
1293 + self._process_sequence_group_outputs(seq_group, output)
1294 continue
1295
1246 - self.output_processor.process_prompt_logprob(seq_group, outputs)
1296 + self.output_processor.process_prompt_logprob(seq_group, output)
1297 if seq_group_meta.do_sample:
1248 - self.output_processor.process_outputs(seq_group, outputs)
1298 + self.output_processor.process_outputs(seq_group, output,
1299 + is_async)
1300
1301 # Free the finished sequence groups.
1302 for scheduler in self.scheduler:
1303 scheduler.free_finished_seq_groups()
1304
1305 # Create the outputs.
1255 - request_outputs: List[Union[RequestOutput,
1256 - EmbeddingRequestOutput]] = []
1257 - for scheduled_seq_group in scheduled_seq_groups:
1306 + for i, _ in enumerate(seq_group_metadata_list):
1307 + scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1308 +
1309 + if i in finished_before:
1310 + continue # Avoids double processing
1311 +
1312 seq_group = scheduled_seq_group.seq_group
1313 seq_group.maybe_set_first_token_time(now)
1314 + if (seq_group.is_finished()
1315 + if self.step_return_finished_only else True):
1316 + request_output = RequestOutputFactory.create(seq_group)
1317 + self.request_outputs.append(request_output)
1318 +
1319 + for seq_group in scheduler_outputs.ignored_seq_groups:
1320 request_output = RequestOutputFactory.create(seq_group)
1261 - request_outputs.append(request_output)
1262 - for seq_group in ignored_seq_groups:
1263 - request_output = RequestOutputFactory.create(seq_group)
1264 - request_outputs.append(request_output)
1265 - return request_outputs
1321 + self.request_outputs.append(request_output)
1322 +
1323 + if is_async:
1324 + # Log stats.
1325 + self.do_log_stats(scheduler_outputs, outputs, finished_before)
1326 +
1327 + # Tracing
1328 + self.do_tracing(scheduler_outputs)
1329 +
1330 + return None
1331 +
1332 + def _advance_to_next_step(
1333 + self, output: List[SamplerOutput],
1334 + seq_group_metadata_list: List[SequenceGroupMetadata],
1335 + scheduled_seq_groups: List[ScheduledSequenceGroup]) -> None:
1336 + """Given model output from a single run, append the tokens to the
1337 + sequences. This is normally done inside output processor, but it is
1338 + required if the worker is to perform async forward pass to next step.
1339 + """
1340 + for seq_group_metadata, sequence_group_outputs, scheduled_seq_group in \
1341 + zip(seq_group_metadata_list, output, scheduled_seq_groups):
1342 + seq_group = scheduled_seq_group.seq_group
1343 +
1344 + if seq_group.is_finished():
1345 + continue
1346 +
1347 + seq_group.update_num_computed_tokens(
1348 + seq_group_metadata.token_chunk_size)
1349 +
1350 + if seq_group_metadata.do_sample:
1351 + assert len(sequence_group_outputs.samples) == 1, (
1352 + "Async output processor expects a single sample"
1353 + " (i.e sampling_params.n == 1 and no "
1354 + "sampling_params.best_of > 1)")
1355 + sample = sequence_group_outputs.samples[0]
1356 +
1357 + assert len(seq_group.seqs) == 1
1358 + seq = seq_group.seqs[0]
1359 + seq.append_token_id(sample.output_token, sample.logprobs)
1360
1361 def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1362 """Performs one decoding iteration and returns newly generated results.
@@ -1325,24 +1419,32 @@ class LLMEngine:
1419 cached_outputs = self.cached_scheduler_outputs[0]
1420 seq_group_metadata_list = cached_outputs.seq_group_metadata_list
1421 scheduler_outputs = cached_outputs.scheduler_outputs
1422 + allow_async_output_proc = cached_outputs.allow_async_output_proc
1423
1424 # Skip the scheduler if there are any remaining steps in the seq groups.
1425 # This ensures that the scheduler is only called again when the current
1426 # batch has completed.
1427 if not self._has_remaining_steps(seq_group_metadata_list):
1333 - seq_group_metadata_list, scheduler_outputs = self.scheduler[
1334 - 0].schedule()
1428 + (seq_group_metadata_list, scheduler_outputs,
1429 + allow_async_output_proc) = self.scheduler[0].schedule()
1430 +
1431 + if not allow_async_output_proc and len(self.output_queue) > 0:
1432 + self._process_model_outputs(is_async=True)
1433
1434 if (self.scheduler_config.is_multi_step
1435 and scheduler_outputs.num_lookahead_slots > 0):
1436 # cache the scheduler outputs for the next iteration if we have
1437 # lookahead slots
1438 self._cache_scheduler_outputs_for_multi_step(
1341 - 0, seq_group_metadata_list, scheduler_outputs)
1439 + 0, seq_group_metadata_list, scheduler_outputs,
1440 + allow_async_output_proc)
1441
1442 assert seq_group_metadata_list is not None
1443 assert scheduler_outputs is not None
1444
1445 + assert not (self.scheduler_config.is_multi_step and \
1446 + allow_async_output_proc)
1447 +
1448 if not scheduler_outputs.is_empty():
1449 finished_requests_ids = self.scheduler[
1450 0].get_and_reset_finished_requests_ids()
@@ -1366,6 +1468,10 @@ class LLMEngine:
1468 # to each of the non-last PP stages for in-place prepare_input.
1469 last_sampled_token_ids=last_sampled_token_ids)
1470
1471 + if allow_async_output_proc:
1472 + execute_model_req.output_proc_callback_fn = \
1473 + self._process_model_outputs
1474 +
1475 output = self.model_executor.execute_model(
1476 execute_model_req=execute_model_req)
1477
@@ -1374,6 +1480,9 @@ class LLMEngine:
1480 if self.scheduler_config.is_multi_step:
1481 self._update_cached_scheduler_output(0, output)
1482 else:
1483 + if len(self.output_queue) > 0:
1484 + assert not self.scheduler_config.is_multi_step
1485 + self._process_model_outputs(is_async=True)
1486 output = []
1487
1488 # Finish the current step for all the sequence groups.
@@ -1382,23 +1491,41 @@ class LLMEngine:
1491 seq_group.finish_step()
1492
1493 if not self._has_remaining_steps(seq_group_metadata_list):
1385 - # clear the cache if we have finished all the steps
1494 + # clear the cache if we have finished all the steps.
1495 if self.scheduler_config.is_multi_step:
1496 self.cached_scheduler_outputs[0] = SchedulerOutputState()
1388 - request_outputs = self._process_model_outputs(
1389 - output, scheduler_outputs.scheduled_seq_groups,
1390 - scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
1497
1392 - else:
1393 - request_outputs = []
1498 + # Add results to the output_queue
1499 + # (for async or non-async postprocessing)
1500 + self.output_queue.append(
1501 + (output, seq_group_metadata_list, scheduler_outputs))
1502
1395 - # Log stats.
1396 - self.do_log_stats(scheduler_outputs, output)
1503 + if output and allow_async_output_proc:
1504 + assert len(output) == 1, ("Multi step decoding does not work "
1505 + "with async output processing.")
1506
1398 - # Tracing
1399 - self.do_tracing(scheduler_outputs)
1507 + self._advance_to_next_step(
1508 + output[0], seq_group_metadata_list,
1509 + scheduler_outputs.scheduled_seq_groups)
1510 +
1511 + if not allow_async_output_proc:
1512 + self._process_model_outputs(is_async=False)
1513 +
1514 + # Log stats.
1515 + self.do_log_stats(scheduler_outputs, output)
1516 +
1517 + # Tracing
1518 + self.do_tracing(scheduler_outputs)
1519 + else:
1520 + self.request_outputs = []
1521
1522 if not self.has_unfinished_requests():
1523 + # Drain async postprocessor
1524 + if len(self.output_queue) > 0:
1525 + assert not self.scheduler_config.is_multi_step
1526 + self._process_model_outputs(is_async=True, clear_outputs=False)
1527 + assert len(self.output_queue) == 0
1528 +
1529 # Stop the execute model loop in parallel workers until there are
1530 # more requests to process. This avoids waiting indefinitely in
1531 # torch.distributed ops which may otherwise timeout, and unblocks
@@ -1406,7 +1533,7 @@ class LLMEngine:
1533 # queued control plane messages, such as add/remove lora adapters.
1534 self.model_executor.stop_remote_worker_execution_loop()
1535
1409 - return request_outputs
1536 + return self.request_outputs
1537
1538 def _has_remaining_steps(
1539 self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
@@ -1431,12 +1558,14 @@ class LLMEngine:
1558 def _cache_scheduler_outputs_for_multi_step(
1559 self, virtual_engine: int,
1560 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
1434 - scheduler_outputs: SchedulerOutputs) -> None:
1435 - self.cached_scheduler_outputs[
1436 - virtual_engine].seq_group_metadata_list = seq_group_metadata_list
1437 - self.cached_scheduler_outputs[virtual_engine].scheduler_outputs = \
1438 - scheduler_outputs
1439 - self.cached_scheduler_outputs[virtual_engine].last_output = None
1561 + scheduler_outputs: SchedulerOutputs,
1562 + allow_async_output_proc: bool) -> None:
1563 + co = self.cached_scheduler_outputs[virtual_engine]
1564 +
1565 + co.seq_group_metadata_list = seq_group_metadata_list
1566 + co.scheduler_outputs = scheduler_outputs
1567 + co.allow_async_output_proc = allow_async_output_proc
1568 + co.last_output = None
1569
1570 def _update_cached_scheduler_output(
1571 self, virtual_engine: int,
@@ -1472,20 +1601,21 @@ class LLMEngine:
1601 raise KeyError(f"Logger with name {logger_name} does not exist.")
1602 del self.stat_loggers[logger_name]
1603
1475 - def do_log_stats(
1476 - self,
1477 - scheduler_outputs: Optional[SchedulerOutputs] = None,
1478 - model_output: Optional[List[SamplerOutput]] = None) -> None:
1604 + def do_log_stats(self,
1605 + scheduler_outputs: Optional[SchedulerOutputs] = None,
1606 + model_output: Optional[List[SamplerOutput]] = None,
1607 + finished_before: Optional[List[int]] = None) -> None:
1608 """Forced log when no requests active."""
1609 if self.log_stats:
1481 - stats = self._get_stats(scheduler_outputs, model_output)
1610 + stats = self._get_stats(scheduler_outputs, model_output,
1611 + finished_before)
1612 for logger in self.stat_loggers.values():
1613 logger.log(stats)
1614
1485 - def _get_stats(
1486 - self,
1487 - scheduler_outputs: Optional[SchedulerOutputs],
1488 - model_output: Optional[List[SamplerOutput]] = None) -> Stats:
1615 + def _get_stats(self,
1616 + scheduler_outputs: Optional[SchedulerOutputs],
1617 + model_output: Optional[List[SamplerOutput]] = None,
1618 + finished_before: Optional[List[int]] = None) -> Stats:
1619 """Get Stats to be Logged to Prometheus.
1620
1621 Args:
@@ -1550,6 +1680,10 @@ class LLMEngine:
1680 # NOTE: This loop assumes prefill seq_groups are before
1681 # decode seq_groups in scheduled_seq_groups.
1682 if scheduler_outputs is not None:
1683 + # For async postprocessor, already finished sequences need to be
1684 + # not counted (to avoid double counting)
1685 + actual_num_batched_tokens = scheduler_outputs.num_batched_tokens # type: ignore
1686 +
1687 num_generation_tokens_from_prefill_groups = 0.
1688 # NOTE: if scheduler_outputs.num_prefill_groups > 0 and
1689 # the len of scheduler_outputs.scheduled_seq_groups is !=
@@ -1558,6 +1692,11 @@ class LLMEngine:
1692
1693 for idx, scheduled_seq_group in enumerate(
1694 scheduler_outputs.scheduled_seq_groups):
1695 + # Skip double logging when using async output proc
1696 + if finished_before and idx in finished_before:
1697 + actual_num_batched_tokens -= 1
1698 + continue
1699 +
1700 group_was_prefill = idx < scheduler_outputs.num_prefill_groups
1701 seq_group = scheduled_seq_group.seq_group
1702
@@ -1592,7 +1731,6 @@ class LLMEngine:
1731 # Latency timings
1732 time_e2e_requests.append(now -
1733 seq_group.metrics.arrival_time)
1595 -
1734 # Metadata
1735 num_prompt_tokens_requests.append(
1736 len(seq_group.prompt_token_ids))
@@ -1616,7 +1754,7 @@ class LLMEngine:
1754 # + num_generation_tokens_from_prefill_groups (since we generate
1755 # one token on prefills on iters where the prefill finishes).
1756 num_generation_tokens_iter = (
1619 - scheduler_outputs.num_batched_tokens - num_prompt_tokens_iter +
1757 + actual_num_batched_tokens - num_prompt_tokens_iter +
1758 num_generation_tokens_from_prefill_groups)
1759
1760 # Spec decode, if enabled, emits specialized metrics from the worker in
vllm/engine/output_processor/interfaces.py
+5 -8
@@ -40,13 +40,9 @@ class SequenceGroupOutputProcessor(ABC):
40 # Importing here to avoid cycle.
41 from vllm.engine.output_processor.single_step import (
42 SingleStepOutputProcessor)
43 - return SingleStepOutputProcessor(
44 - scheduler_config,
45 - detokenizer,
46 - scheduler,
47 - seq_counter,
48 - stop_checker,
49 - )
43 + return SingleStepOutputProcessor(scheduler_config, detokenizer,
44 + scheduler, seq_counter,
45 + stop_checker)
46 else:
47 # Importing here to avoid cycle.
48 from vllm.engine.output_processor.multi_step import (
@@ -61,7 +57,8 @@ class SequenceGroupOutputProcessor(ABC):
57
58 @abstractmethod
59 def process_outputs(self, sequence_group: SequenceGroup,
64 - outputs: List[SequenceGroupOutput]) -> None:
60 + outputs: List[SequenceGroupOutput],
61 + is_async: bool) -> None:
62 """Process new token ids for the sequence group. Handles logic such as
63 detokenization, stop checking, and freeing/forking sequences in the
64 scheduler.
vllm/engine/output_processor/multi_step.py
+16 -9
@@ -57,17 +57,28 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
57 "Prompt logprob is not supported by multi step workers. "
58 "(e.g., speculative decode uses multi step workers).")
59
60 - def process_outputs(self, sequence_group: SequenceGroup,
61 - outputs: List[SequenceGroupOutput]) -> None:
60 + def process_outputs(self,
61 + sequence_group: SequenceGroup,
62 + outputs: List[SequenceGroupOutput],
63 + is_async: bool = False) -> None:
64 """Append new tokens in the outputs to sequences in the sequence group.
65
66 This only supports sequence groups of size 1. It supports greater than
67 one new token per sequence.
68
67 - This applies logic like stop condition checking and detokenization,
68 - including freeing finished sequences. It also handles cases where there
69 - are tokens emitted after the EOS token.
69 + This applies logic like stop condition checking and detokenization.
70 + It also handles cases where there are tokens emitted after
71 + the EOS token.
72 +
73 + is_async - Indicates whether this postprocessor runs in
74 + parallel with the GPU forward pass and is processing
75 + tokens from the previous step. If this is true, then
76 + no tokens need to be appended since it is already done
77 + externally (before the next schedule() call)
78 """
79 + # TODO: Add support for async if necessary
80 + assert not is_async
81 +
82 seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING)
83
84 assert seqs, "expected running sequences"
@@ -138,7 +149,3 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
149 )
150 if seq.is_finished():
151 break
141 -
142 - if seq.is_finished():
143 - for scheduler in self.scheduler:
144 - scheduler.free_seq(seq)
vllm/engine/output_processor/single_step.py
+20 -12
@@ -29,14 +29,9 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
29 that is currently difficult to schedule multiple steps ahead of time.
30 """
31
32 - def __init__(
33 - self,
34 - scheduler_config: SchedulerConfig,
35 - detokenizer: Detokenizer,
36 - scheduler: List[Scheduler],
37 - seq_counter: Counter,
38 - stop_checker: StopChecker,
39 - ):
32 + def __init__(self, scheduler_config: SchedulerConfig,
33 + detokenizer: Detokenizer, scheduler: List[Scheduler],
34 + seq_counter: Counter, stop_checker: StopChecker):
35 self.scheduler_config = scheduler_config
36 self.detokenizer = detokenizer
37 self.scheduler = scheduler
@@ -44,16 +39,24 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
39 self.stop_checker = stop_checker
40
41 def process_outputs(self, sequence_group: SequenceGroup,
47 - outputs: List[SequenceGroupOutput]) -> None:
42 + outputs: List[SequenceGroupOutput],
43 + is_async: bool) -> None:
44 """Append all new tokens to sequences in the sequence group. Fork any
45 surviving beam candidates; free any unsurviving ones.
46
47 Invokes detokenizer to detokenize new tokens, and also marks sequences
48 as finished if they meet stop conditions.
49 +
50 + is_async - Indicates whether this postprocessor runs in
51 + parallel with the GPU forward pass and is processing
52 + tokens from the previous step. If this is true, then
53 + no tokens need to be appended since it is already done
54 + externally (before the next schedule() call)
55 """
56 assert (len(outputs) == 1
57 ), f"{type(self)} does not support multiple outputs per step"
56 - return self._process_sequence_group_outputs(sequence_group, outputs[0])
58 + return self._process_sequence_group_outputs(sequence_group, outputs[0],
59 + is_async)
60
61 def process_prompt_logprob(self, seq_group: SequenceGroup,
62 outputs: List[SequenceGroupOutput]) -> None:
@@ -80,14 +83,16 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
83 seq_group.prompt_logprobs.extend(prompt_logprobs)
84
85 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
83 - outputs: SequenceGroupOutput) -> None:
86 + outputs: SequenceGroupOutput,
87 + is_async: bool) -> None:
88 sampling_params = seq_group.sampling_params
89 if sampling_params.n == 1 and not sampling_params.use_beam_search:
90 # only have one output sample
91 sample = outputs.samples[0]
92 # only have one sequence
93 seq = seq_group.seqs[0]
90 - seq.append_token_id(sample.output_token, sample.logprobs)
94 + if not is_async:
95 + seq.append_token_id(sample.output_token, sample.logprobs)
96 if sampling_params.detokenize and self.detokenizer:
97 new_char_count = self.detokenizer.decode_sequence_inplace(
98 seq, sampling_params)
@@ -104,6 +109,9 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
109 scheduler.free_seq(seq)
110 return
111
112 + # TODO: Add support for async for beam search
113 + assert not is_async
114 +
115 # Process samples
116 samples = outputs.samples
117 parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
vllm/entrypoints/llm.py
+10 -1
@@ -129,6 +129,7 @@ class LLM:
129 max_context_len_to_capture: Optional[int] = None,
130 max_seq_len_to_capture: int = 8192,
131 disable_custom_all_reduce: bool = False,
132 + disable_async_output_proc: bool = False,
133 **kwargs,
134 ) -> None:
135 '''
@@ -170,6 +171,7 @@ class LLM:
171 max_context_len_to_capture=max_context_len_to_capture,
172 max_seq_len_to_capture=max_seq_len_to_capture,
173 disable_custom_all_reduce=disable_custom_all_reduce,
174 + disable_async_output_proc=disable_async_output_proc,
175 **kwargs,
176 )
177 self.llm_engine = LLMEngine.from_engine_args(
@@ -603,7 +605,6 @@ class LLM:
605 inputs = [inputs]
606
607 num_requests = len(inputs)
606 -
608 if isinstance(params, list) and len(params) != num_requests:
609 raise ValueError("The lengths of prompts and params "
610 "must be the same.")
@@ -678,6 +679,10 @@ class LLM:
679 postfix=(f"est. speed input: {0:.2f} toks/s, "
680 f"output: {0:.2f} toks/s"),
681 )
682 +
683 + # In the loop below, only finished outputs are used
684 + self.llm_engine.step_return_finished_only = True
685 +
686 # Run the engine.
687 outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
688 total_in_toks = 0
@@ -700,6 +705,10 @@ class LLM:
705 f"est. speed input: {in_spd:.2f} toks/s, "
706 f"output: {out_spd:.2f} toks/s")
707 pbar.update(1)
708 +
709 + # Restore original behavior
710 + self.llm_engine.step_return_finished_only = False
711 +
712 if use_tqdm:
713 pbar.close()
714 # Sort the outputs by request ID.
vllm/executor/distributed_gpu_executor.py
+4 -3
@@ -64,8 +64,9 @@ class DistributedGPUExecutor(GPUExecutor):
64 num_cpu_blocks=num_cpu_blocks)
65
66 def execute_model(
67 - self,
68 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
67 + self,
68 + execute_model_req: ExecuteModelRequest,
69 + ) -> List[SamplerOutput]:
70 if self.parallel_worker_tasks is None:
71 self.parallel_worker_tasks = self._run_workers(
72 "start_worker_execution_loop",
@@ -188,7 +189,7 @@ class DistributedGPUExecutorAsync(DistributedGPUExecutor, ExecutorAsyncBase):
189 @abstractmethod
190 async def _driver_execute_model_async(
191 self,
191 - execute_model_req: Optional[ExecuteModelRequest] = None
192 + execute_model_req: Optional[ExecuteModelRequest] = None,
193 ) -> List[SamplerOutput]:
194 """Execute the model asynchronously in the driver worker.
195
vllm/executor/gpu_executor.py
+1 -1
@@ -176,5 +176,5 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
176 execute_model_req: ExecuteModelRequest,
177 ) -> List[Union[SamplerOutput, PoolerOutput]]:
178 output = await make_async(self.driver_worker.execute_model
179 - )(execute_model_req=execute_model_req, )
179 + )(execute_model_req=execute_model_req)
180 return output
vllm/sequence.py
+8 -8
@@ -5,8 +5,8 @@ from abc import ABC, abstractmethod
5 from array import array
6 from collections import defaultdict
7 from dataclasses import dataclass
8 -from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set,
9 - Tuple, Union, cast)
8 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
9 + Optional, Set, Tuple, Union, cast)
10
11 import msgspec
12 import torch
@@ -474,11 +474,8 @@ class Sequence:
474 """Reset the sequence states for recomputation."""
475 self.data.reset_state_for_recompute()
476
477 - def append_token_id(
478 - self,
479 - token_id: int,
480 - logprobs: Dict[int, Logprob],
481 - ) -> None:
477 + def append_token_id(self, token_id: int, logprobs: Dict[int,
478 + Logprob]) -> None:
479 assert token_id in logprobs
480 self.output_logprobs.append(logprobs)
481 self.data.append_token_id(token_id, logprobs[token_id].logprob)
@@ -1293,6 +1290,8 @@ class ExecuteModelRequest(
1290 finished_requests_ids: List[str] = msgspec.field(default_factory=list)
1291 # The last sampled token ids for multi step decoding.
1292 last_sampled_token_ids: Optional[torch.Tensor] = None
1293 + # Async postprocessor
1294 + output_proc_callback_fn: Optional[Callable] = None
1295
1296 @property
1297 def is_first_multi_step(self) -> bool:
@@ -1338,4 +1337,5 @@ class ExecuteModelRequest(
1337 num_steps=self.num_steps,
1338 finished_requests_ids=self.finished_requests_ids,
1339 last_sampled_token_ids=self.last_sampled_token_ids.clone()
1341 - if self.last_sampled_token_ids is not None else None)
1340 + if self.last_sampled_token_ids is not None else None,
1341 + output_proc_callback_fn=self.output_proc_callback_fn)
vllm/worker/model_runner.py
+7 -3
@@ -6,8 +6,8 @@ import time
6 import warnings
7 import weakref
8 from dataclasses import dataclass
9 -from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type,
10 - TypeVar, Union)
9 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set,
10 + Tuple, Type, TypeVar, Union)
11
12 import numpy as np
13 import torch
@@ -90,6 +90,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
90 request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
91 finished_requests_ids: Optional[List[str]] = None
92 virtual_engine: int = 0
93 + output_proc_callback_fn: Optional[Callable] = None
94
95 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
96 tensor_dict = {
@@ -1327,7 +1328,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1328 self,
1329 seq_group_metadata_list: List[SequenceGroupMetadata],
1330 virtual_engine: int = 0,
1330 - finished_requests_ids: Optional[List[str]] = None
1331 + finished_requests_ids: Optional[List[str]] = None,
1332 ) -> ModelInputForGPUWithSamplingMetadata:
1333 """Prepare the model input based on a given sequence group, including
1334 metadata for the sampling step.
@@ -1451,6 +1452,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1452 if not self.is_driver_worker:
1453 return []
1454
1455 + if model_input.output_proc_callback_fn is not None:
1456 + model_input.output_proc_callback_fn(is_async=True)
1457 +
1458 # Sample the next token.
1459 output: SamplerOutput = self.model.sample(
1460 logits=logits,
vllm/worker/worker_base.py
+7 -1
@@ -263,6 +263,12 @@ class LocalOrDistributedWorkerBase(WorkerBase):
263 broadcast_data.update(kwargs)
264 broadcast_tensor_dict(broadcast_data, src=0)
265
266 + if execute_model_req.output_proc_callback_fn:
267 + model_input = dataclasses.replace( # type: ignore
268 + model_input,
269 + output_proc_callback_fn=execute_model_req.
270 + output_proc_callback_fn)
271 +
272 return model_input, worker_input, kwargs
273
274 def prepare_input(
@@ -289,7 +295,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
295
296 def execute_model(
297 self,
292 - execute_model_req: Optional[ExecuteModelRequest] = None
298 + execute_model_req: Optional[ExecuteModelRequest] = None,
299 ) -> Optional[List[SamplerOutput]]:
300 """Executes at least one model step on the given sequences, unless no
301 sequences are provided."""