@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f60f2244e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core] Combine async postprocessor and multi-step (#7921)

Alexander Matveev committed Aug 29, 2024 at 14:18 UTC 3f60f2244e3ffec6198d7a41765918d1efd3bb96
8 files changed +215 -65
tests/multi_step/test_correctness_async_llm.py
+6 -4
@@ -47,10 +47,12 @@ async def completions_with_server_args(prompts: List[str], model_name: str,
47 @pytest.mark.parametrize("eager_mode", [False, True])
48 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
49 @pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
50 +@pytest.mark.parametrize("is_async", [False, True])
51 @pytest.mark.asyncio
52 async def test_multi_step(example_prompts, model: str, tp_size: int,
53 pp_size: int, eager_mode: int,
53 - num_scheduler_steps: int, num_prompts: int):
54 + num_scheduler_steps: int, num_prompts: int,
55 + is_async: bool):
56
57 prompts = example_prompts
58 if len(prompts) < num_prompts:
@@ -62,9 +64,9 @@ async def test_multi_step(example_prompts, model: str, tp_size: int,
64 ms_server_args = DEFAULT_SERVER_ARGS + \
65 ["--num-scheduler-steps", f"{num_scheduler_steps}"]
66
65 - # Disable output proc callback as its not supported
66 - # with multi-step right now
67 - ms_server_args += ["--disable-async-output-proc"]
67 + if not is_async:
68 + ms_server_args += ["--disable-async-output-proc"]
69 +
70 if eager_mode:
71 ms_server_args.append("--enforce-eager")
72
vllm/core/scheduler.py
+1 -4
@@ -1107,10 +1107,7 @@ class Scheduler:
1107 if not self.cache_config.enable_prefix_caching:
1108 common_computed_block_nums = []
1109
1110 - # TODO: Combine multi-step and async postprocessor
1111 - allow_async_output_proc: bool = (
1112 - self.use_async_output_proc
1113 - and not self.scheduler_config.is_multi_step)
1110 + allow_async_output_proc: bool = self.use_async_output_proc
1111
1112 # Create input data structures.
1113 seq_group_metadata_list: List[SequenceGroupMetadata] = []
vllm/engine/async_llm_engine.py
+44 -21
@@ -279,6 +279,10 @@ class _AsyncLLMEngine(LLMEngine):
279 scheduler_outputs = cached_outputs.scheduler_outputs
280 allow_async_output_proc = cached_outputs.allow_async_output_proc
281
282 + # Detect async + multi-step
283 + use_async_and_multi_step = (self.scheduler_config.is_multi_step
284 + and allow_async_output_proc)
285 +
286 ctx = self.scheduler_contexts[virtual_engine]
287
288 # skip the scheduler if there are any remaining steps in the seq groups.
@@ -289,17 +293,27 @@ class _AsyncLLMEngine(LLMEngine):
293 # Clear outputs on scheduler iteration start
294 ctx.request_outputs.clear()
295
296 + # Schedule iteration
297 (seq_group_metadata_list, scheduler_outputs,
298 allow_async_output_proc
299 ) = self.scheduler[virtual_engine].schedule()
300
296 - # If current scheduler iteration has no async postprocessor,
297 - # then we need first to drain the pending async postprocessor
298 - # before moving forward
301 + # Detect async + multi-step
302 + use_async_and_multi_step = (self.scheduler_config.is_multi_step
303 + and allow_async_output_proc)
304 +
305 + # Maybe switch from async mode to sync mode
306 if not allow_async_output_proc and len(ctx.output_queue) > 0:
307 self._process_model_outputs(virtual_engine=virtual_engine,
308 is_async=True)
309
310 + # For async + multi-step, init the queue
311 + if use_async_and_multi_step:
312 + assert len(ctx.output_queue) == 0
313 + assert seq_group_metadata_list is not None
314 + ctx.output_queue.append(
315 + (None, seq_group_metadata_list, scheduler_outputs))
316 +
317 if (self.scheduler_config.is_multi_step
318 and scheduler_outputs.num_lookahead_slots > 0):
319 # cache the scheduler outputs for the next iteration if we have
@@ -311,9 +325,6 @@ class _AsyncLLMEngine(LLMEngine):
325 assert seq_group_metadata_list is not None
326 assert scheduler_outputs is not None
327
314 - assert not (self.scheduler_config.is_multi_step and \
315 - allow_async_output_proc)
316 -
328 if not scheduler_outputs.is_empty():
329 finished_requests_ids = self.scheduler[
330 virtual_engine].get_and_reset_finished_requests_ids()
@@ -339,8 +350,13 @@ class _AsyncLLMEngine(LLMEngine):
350 last_sampled_token_ids=last_sampled_token_ids)
351
352 if allow_async_output_proc:
342 - execute_model_req.async_callback = self.async_callback[
343 - virtual_engine]
353 + async_callback = self.async_callback_multi_step[
354 + virtual_engine] if use_async_and_multi_step \
355 + else self.async_callback[virtual_engine]
356 +
357 + execute_model_req.async_callback = async_callback
358 + execute_model_req.use_async_and_multi_step = \
359 + use_async_and_multi_step
360
361 # Execute the model.
362 output = await self.model_executor.execute_model_async(
@@ -350,7 +366,7 @@ class _AsyncLLMEngine(LLMEngine):
366 if self.scheduler_config.is_multi_step:
367 self._update_cached_scheduler_output(virtual_engine, output)
368 else:
353 - if len(ctx.output_queue) > 0:
369 + if not use_async_and_multi_step and len(ctx.output_queue) > 0:
370 assert not self.scheduler_config.is_multi_step
371 self._process_model_outputs(virtual_engine=virtual_engine,
372 is_async=True)
@@ -362,22 +378,25 @@ class _AsyncLLMEngine(LLMEngine):
378 seq_group.finish_step()
379
380 if not self._has_remaining_steps(seq_group_metadata_list):
365 - # clear the cache if we have finished all the steps
381 + # Clear the cache if we have finished all the steps
382 if self.scheduler_config.is_multi_step:
383 self.cached_scheduler_outputs[
384 virtual_engine] = SchedulerOutputState()
385
370 - # Cache results in engine
371 - ctx.output_queue.append(
372 - (output, seq_group_metadata_list, scheduler_outputs))
386 + if use_async_and_multi_step:
387 + # For async + multi-step, clear the queue
388 + ctx.output_queue.clear()
389 + else:
390 + ctx.output_queue.append(
391 + (output, seq_group_metadata_list, scheduler_outputs))
392
374 - if output and allow_async_output_proc:
375 - assert len(
376 - output
377 - ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
378 - self._advance_to_next_step(
379 - output[0], seq_group_metadata_list,
380 - scheduler_outputs.scheduled_seq_groups)
393 + if output and allow_async_output_proc:
394 + assert len(
395 + output
396 + ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
397 + self._advance_to_next_step(
398 + output[0], seq_group_metadata_list,
399 + scheduler_outputs.scheduled_seq_groups)
400
401 if not allow_async_output_proc:
402 self._process_model_outputs(virtual_engine=virtual_engine,
@@ -390,7 +409,11 @@ class _AsyncLLMEngine(LLMEngine):
409 self.do_tracing(scheduler_outputs)
410
411 else:
393 - ctx.request_outputs = []
412 + # Multi-step case
413 + if use_async_and_multi_step:
414 + return []
415 + else:
416 + ctx.request_outputs = []
417
418 if not self.has_unfinished_requests():
419 # Drain async postprocessor (if exists)
vllm/engine/llm_engine.py
+88 -26
@@ -91,7 +91,8 @@ class SchedulerOutputState:
91
92 @dataclass
93 class SchedulerContext:
94 - output_queue: Deque[Tuple[List[SamplerOutput], List[SequenceGroupMetadata],
94 + output_queue: Deque[Tuple[Optional[List[SamplerOutput]],
95 + List[SequenceGroupMetadata],
96 SchedulerOutputs]] = field(
97 default_factory=lambda: deque())
98
@@ -432,6 +433,13 @@ class LLMEngine:
433 for v_id in range(self.parallel_config.pipeline_parallel_size)
434 ]
435
436 + self.async_callback_multi_step = [
437 + functools.partial(self._process_model_outputs,
438 + virtual_engine=v_id,
439 + is_async=False)
440 + for v_id in range(self.parallel_config.pipeline_parallel_size)
441 + ]
442 +
443 def _initialize_kv_caches(self) -> None:
444 """Initialize the KV cache in the worker(s).
445
@@ -1240,28 +1248,49 @@ class LLMEngine:
1248
1249 return
1250
1243 - def _process_model_outputs(self, virtual_engine: int,
1244 - is_async: bool) -> None:
1251 + def _process_model_outputs(self,
1252 + virtual_engine: int,
1253 + is_async: bool,
1254 + sampler_output: Optional[SamplerOutput] = None,
1255 + is_last_output: bool = False) -> None:
1256 """Apply the model output to the sequences in the scheduled seq groups.
1257
1258 virtual_engine: The engine id to operate on
1259 +
1260 is_async: Indicates whether this postprocessor runs in
1261 parallel with the GPU forward pass and is processing
1262 tokens from the previous step. If this is true, then
1263 no tokens need to be appended since it is already done
1264 externally (before the next schedule() call)
1265
1266 + sampler_output: Used with multi-step execution to provide
1267 + sampler_output of each step
1268 + is_last_output: Used with multi-step execution to indicate
1269 + the last step (of each multi-step group)
1270 +
1271 Returns RequestOutputs that can be returned to the client.
1272 """
1273 now = time.time()
1274
1275 + is_multi_step = sampler_output is not None
1276 +
1277 ctx: SchedulerContext = self.scheduler_contexts[virtual_engine]
1278
1279 if len(ctx.output_queue) == 0:
1280 return None
1281
1263 - (outputs, seq_group_metadata_list,
1264 - scheduler_outputs) = ctx.output_queue.popleft()
1282 + if is_multi_step:
1283 + # Async + multi-step case
1284 + (outputs, seq_group_metadata_list,
1285 + scheduler_outputs) = ctx.output_queue[0]
1286 + assert outputs is None
1287 + outputs = [sampler_output]
1288 + else:
1289 + # Async standard case
1290 + (outputs, seq_group_metadata_list,
1291 + scheduler_outputs) = ctx.output_queue.popleft()
1292 +
1293 + assert outputs is not None
1294
1295 # Sanity check
1296 assert len(seq_group_metadata_list) == len(
@@ -1320,7 +1349,11 @@ class LLMEngine:
1349 self.output_processor.process_outputs(seq_group, output,
1350 is_async)
1351
1323 - # Free the finished sequence groups.
1352 + # For async + multi-step, free finished seqs and create outputs
1353 + # only on the final step.
1354 + if is_multi_step and not is_last_output:
1355 + return
1356 +
1357 for scheduler in self.scheduler:
1358 scheduler.free_finished_seq_groups()
1359
@@ -1328,7 +1361,7 @@ class LLMEngine:
1361 for i, _ in enumerate(seq_group_metadata_list):
1362 scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1363
1331 - if i in finished_before:
1364 + if not is_multi_step and i in finished_before:
1365 continue # Avoids double processing
1366
1367 seq_group = scheduled_seq_group.seq_group
@@ -1342,7 +1375,11 @@ class LLMEngine:
1375 request_output = RequestOutputFactory.create(seq_group)
1376 ctx.request_outputs.append(request_output)
1377
1345 - if is_async:
1378 + # For async + multi-step, do stats only on the last output.
1379 + # Otherwise, do stats if the execution is async
1380 + do_stats = is_multi_step or is_async
1381 +
1382 + if do_stats:
1383 # Log stats.
1384 self.do_log_stats(scheduler_outputs, outputs, finished_before)
1385
@@ -1437,7 +1474,7 @@ class LLMEngine:
1474 "as performance will be severely degraded otherwise.")
1475
1476 # For llm_engine, there is no pipeline parallel support, so the engine
1440 - # used is always 0
1477 + # used is always 0.
1478 virtual_engine = 0
1479
1480 # These are cached outputs from previous iterations. None if on first
@@ -1447,6 +1484,10 @@ class LLMEngine:
1484 scheduler_outputs = cached_outputs.scheduler_outputs
1485 allow_async_output_proc = cached_outputs.allow_async_output_proc
1486
1487 + # Detect async + multi-step
1488 + use_async_and_multi_step = (self.scheduler_config.is_multi_step
1489 + and allow_async_output_proc)
1490 +
1491 ctx = self.scheduler_contexts[virtual_engine]
1492
1493 # Skip the scheduler if there are any remaining steps in the seq groups.
@@ -1462,11 +1503,22 @@ class LLMEngine:
1503 allow_async_output_proc
1504 ) = self.scheduler[virtual_engine].schedule()
1505
1506 + # Detect async + multi-step
1507 + use_async_and_multi_step = (self.scheduler_config.is_multi_step
1508 + and allow_async_output_proc)
1509 +
1510 # Maybe switch from async mode to sync mode
1511 if not allow_async_output_proc and len(ctx.output_queue) > 0:
1512 self._process_model_outputs(virtual_engine=virtual_engine,
1513 is_async=True)
1514
1515 + # For async + multi-step, init the queue
1516 + if use_async_and_multi_step:
1517 + assert len(ctx.output_queue) == 0
1518 + assert seq_group_metadata_list is not None
1519 + ctx.output_queue.append(
1520 + (None, seq_group_metadata_list, scheduler_outputs))
1521 +
1522 if (self.scheduler_config.is_multi_step
1523 and scheduler_outputs.num_lookahead_slots > 0):
1524 # cache the scheduler outputs for the next iteration if we have
@@ -1478,9 +1530,6 @@ class LLMEngine:
1530 assert seq_group_metadata_list is not None
1531 assert scheduler_outputs is not None
1532
1481 - assert not (self.scheduler_config.is_multi_step and \
1482 - allow_async_output_proc)
1483 -
1533 if not scheduler_outputs.is_empty():
1534 finished_requests_ids = self.scheduler[
1535 virtual_engine].get_and_reset_finished_requests_ids()
@@ -1505,8 +1554,13 @@ class LLMEngine:
1554 last_sampled_token_ids=last_sampled_token_ids)
1555
1556 if allow_async_output_proc:
1508 - execute_model_req.async_callback = self.async_callback[
1509 - virtual_engine]
1557 + async_callback = self.async_callback_multi_step[
1558 + virtual_engine] if use_async_and_multi_step \
1559 + else self.async_callback[virtual_engine]
1560 +
1561 + execute_model_req.async_callback = async_callback
1562 + execute_model_req.use_async_and_multi_step = \
1563 + use_async_and_multi_step
1564
1565 output = self.model_executor.execute_model(
1566 execute_model_req=execute_model_req)
@@ -1518,7 +1572,7 @@ class LLMEngine:
1572 else:
1573 # Nothing scheduled => If there is pending async postprocessor,
1574 # then finish it here.
1521 - if len(ctx.output_queue) > 0:
1575 + if not use_async_and_multi_step and len(ctx.output_queue) > 0:
1576 assert not self.scheduler_config.is_multi_step
1577 self._process_model_outputs(virtual_engine=virtual_engine,
1578 is_async=True)
@@ -1535,18 +1589,23 @@ class LLMEngine:
1589 if self.scheduler_config.is_multi_step:
1590 self.cached_scheduler_outputs[0] = SchedulerOutputState()
1591
1538 - # Add results to the output_queue
1539 - # (for async or non-async postprocessing)
1540 - ctx.output_queue.append(
1541 - (output, seq_group_metadata_list, scheduler_outputs))
1592 + if use_async_and_multi_step:
1593 + # For async + multi-step, clear the queue
1594 + ctx.output_queue.clear()
1595 + else:
1596 + # Add results to the output_queue
1597 + # (for async or non-async postprocessing)
1598 + ctx.output_queue.append(
1599 + (output, seq_group_metadata_list, scheduler_outputs))
1600
1543 - if output and allow_async_output_proc:
1544 - assert len(output) == 1, ("Multi step decoding does not work "
1545 - "with async output processing.")
1601 + if output and allow_async_output_proc:
1602 + assert len(output) == 1, (
1603 + "Multi step decoding does not work "
1604 + "with async output processing.")
1605
1547 - self._advance_to_next_step(
1548 - output[0], seq_group_metadata_list,
1549 - scheduler_outputs.scheduled_seq_groups)
1606 + self._advance_to_next_step(
1607 + output[0], seq_group_metadata_list,
1608 + scheduler_outputs.scheduled_seq_groups)
1609
1610 # Check if need to run the usual non-async path
1611 if not allow_async_output_proc:
@@ -1560,7 +1619,10 @@ class LLMEngine:
1619 self.do_tracing(scheduler_outputs)
1620 else:
1621 # Multi-step case
1563 - ctx.request_outputs = []
1622 + if use_async_and_multi_step:
1623 + return []
1624 + else:
1625 + ctx.request_outputs = []
1626
1627 if not self.has_unfinished_requests():
1628 # Drain async postprocessor (if exists)
vllm/sequence.py
+3 -1
@@ -1295,6 +1295,7 @@ class ExecuteModelRequest(
1295 last_sampled_token_ids: Optional[torch.Tensor] = None
1296 # Async callback
1297 async_callback: Optional[Callable] = None
1298 + use_async_and_multi_step: bool = False
1299
1300 @property
1301 def is_first_multi_step(self) -> bool:
@@ -1341,4 +1342,5 @@ class ExecuteModelRequest(
1342 finished_requests_ids=self.finished_requests_ids,
1343 last_sampled_token_ids=self.last_sampled_token_ids.clone()
1344 if self.last_sampled_token_ids is not None else None,
1344 - async_callback=self.async_callback)
1345 + async_callback=self.async_callback,
1346 + use_async_and_multi_step=self.use_async_and_multi_step)
vllm/worker/model_runner.py
+1
@@ -92,6 +92,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
92 finished_requests_ids: Optional[List[str]] = None
93 virtual_engine: int = 0
94 async_callback: Optional[Callable] = None
95 + use_async_and_multi_step: bool = False
96
97 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
98 tensor_dict = {
vllm/worker/multi_step_model_runner.py
+64 -9
@@ -1,5 +1,7 @@
1 +import dataclasses
2 +import functools
3 from dataclasses import dataclass, field
2 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
4 +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
5
6 try:
7 from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -215,6 +217,46 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
217 )
218 return model_input
219
220 + def _async_process_outputs(self, model_input: StatefulModelInput,
221 + output_proc_callback: Callable):
222 + # Proceed with pythonization and output_proc in order.
223 + # Stop on the first one that fails to pythonize
224 + cont = True
225 + for model_output in model_input.cached_outputs:
226 + if not model_output.pythonized:
227 + model_output.maybe_pythonize(model_input, self._copy_stream,
228 + self.pinned_sampled_token_ids)
229 + if model_output.pythonized:
230 + output_proc_callback(
231 + sampler_output=model_output.sampler_output)
232 + else:
233 + cont = False
234 +
235 + if not cont:
236 + break
237 +
238 + def _final_process_outputs(self, model_input: StatefulModelInput,
239 + output_proc_callback: Optional[Callable]):
240 + assert model_input.frozen_model_input is not None
241 +
242 + outputs = []
243 + for output_id in range(len(model_input.cached_outputs)):
244 + is_last_output = output_id == len(model_input.cached_outputs) - 1
245 +
246 + output = model_input.cached_outputs[output_id]
247 + if not output.pythonized:
248 + output.pythonize(model_input, self._copy_stream,
249 + self.pinned_sampled_token_ids)
250 +
251 + if model_input.frozen_model_input.use_async_and_multi_step:
252 + assert output_proc_callback is not None
253 + output_proc_callback(sampler_output=output.sampler_output,
254 + is_last_output=is_last_output)
255 +
256 + outputs.append(output.sampler_output)
257 +
258 + return outputs
259 +
260 @torch.inference_mode()
261 def execute_model(
262 self,
@@ -271,6 +313,20 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
313 model_input = self._advance_step(
314 model_input, model_input.cached_outputs[-1].sampler_output)
315
316 + output_proc_callback = None
317 + if frozen_model_input.use_async_and_multi_step:
318 + output_proc_callback = frozen_model_input.async_callback
319 + assert output_proc_callback is not None
320 + async_callback = functools.partial(
321 + self._async_process_outputs,
322 + model_input=model_input,
323 + output_proc_callback=output_proc_callback)
324 +
325 + frozen_model_input = dataclasses.replace( # type: ignore
326 + model_input.frozen_model_input,
327 + async_callback=async_callback)
328 + assert frozen_model_input is not None
329 +
330 # Execute the model
331 output = self._base_model_runner.execute_model(frozen_model_input,
332 kv_caches,
@@ -301,9 +357,11 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
357 output[0].logprobs = None
358 # Pythonize the output if CPU is ahead and the previous step is
359 # ready.
304 - for model_output in model_input.cached_outputs:
305 - model_output.maybe_pythonize(model_input, self._copy_stream,
306 - self.pinned_sampled_token_ids)
360 + if not frozen_model_input.use_async_and_multi_step:
361 + for model_output in model_input.cached_outputs:
362 + model_output.maybe_pythonize(model_input,
363 + self._copy_stream,
364 + self.pinned_sampled_token_ids)
365
366 model_input.current_step += 1
367
@@ -316,11 +374,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
374
375 # Pythonize the output and block if needed since it is the last step
376 if model_input.is_last_step:
319 - outputs = []
320 - for output in model_input.cached_outputs:
321 - output.pythonize(model_input, self._copy_stream,
322 - self.pinned_sampled_token_ids)
323 - outputs.append(output.sampler_output)
377 + outputs = self._final_process_outputs(model_input,
378 + output_proc_callback)
379 return outputs
380
381 # should be [SamplerOutput]
vllm/worker/multi_step_worker.py
+8
@@ -1,3 +1,4 @@
1 +import dataclasses
2 from dataclasses import dataclass
3 from typing import Dict, List, Optional, Tuple
4
@@ -61,6 +62,13 @@ class MultiStepWorker(Worker):
62 execute_model_req.seq_group_metadata_list,
63 execute_model_req.virtual_engine,
64 execute_model_req.finished_requests_ids))
65 +
66 + if execute_model_req.async_callback:
67 + model_input.frozen_model_input = dataclasses.replace( # type: ignore
68 + model_input.frozen_model_input,
69 + async_callback=execute_model_req.async_callback,
70 + use_async_and_multi_step=execute_model_req.
71 + use_async_and_multi_step)
72 else:
73 # on subsequent steps we reuse the worker input and model input
74 multi_step_state = self.multi_step_states[virtual_engine]