@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f508e03e7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Async_output_proc: Add virtual engine support (towards pipeline parallel) (#7911)

Alexander Matveev committed Aug 28, 2024 at 03:02 UTC f508e03e7f2d8aed897d8843e1ed1668e5c4ad7a
6 files changed +122 -67
vllm/core/scheduler.py
+5 -6
@@ -302,7 +302,7 @@ class Scheduler:
302 cache_config: CacheConfig,
303 lora_config: Optional[LoRAConfig],
304 pipeline_parallel_size: int = 1,
305 - output_proc_callback_fn: Optional[Callable] = None,
305 + output_proc_callback: Optional[Callable] = None,
306 ) -> None:
307 self.scheduler_config = scheduler_config
308 self.cache_config = cache_config
@@ -376,8 +376,8 @@ class Scheduler:
376 # iterations. I.e. since the output processing is lagged one step,
377 # we cannot reuse the cached objects immediately when the schedule()
378 # is called again, but only when schedule() is called the second time.
379 - self.output_proc_callback_fn = output_proc_callback_fn
380 - self.use_async_output_proc = self.output_proc_callback_fn is not None
379 + self.output_proc_callback = output_proc_callback
380 + self.use_async_output_proc = self.output_proc_callback is not None
381 self.num_cache_iters = 2 if self.use_async_output_proc else 1
382
383 self.cache_id = 0
@@ -573,8 +573,8 @@ class Scheduler:
573 seq_group):
574 tmp = self.running
575 self.running = orig_running
576 - assert self.output_proc_callback_fn is not None
577 - self.output_proc_callback_fn(is_async=True)
576 + assert self.output_proc_callback is not None
577 + self.output_proc_callback()
578 self.running = tmp
579
580 while not self._can_append_slots(seq_group):
@@ -1091,7 +1091,6 @@ class Scheduler:
1091 no_beam_search = seq_group.sampling_params is None or (
1092 seq_group.sampling_params.best_of == 1
1093 and not seq_group.sampling_params.use_beam_search)
1094 -
1094 return no_beam_search
1095
1096 def schedule(
vllm/engine/async_llm_engine.py
+27 -10
@@ -279,10 +279,16 @@ class _AsyncLLMEngine(LLMEngine):
279 scheduler_outputs = cached_outputs.scheduler_outputs
280 allow_async_output_proc = cached_outputs.allow_async_output_proc
281
282 + ctx = self.scheduler_contexts[virtual_engine]
283 +
284 # skip the scheduler if there are any remaining steps in the seq groups.
285 # This ensures that the scheduler is only called again when the current
286 # batch has completed.
287 if not self._has_remaining_steps(seq_group_metadata_list):
288 +
289 + # Clear outputs on scheduler iteration start
290 + ctx.request_outputs.clear()
291 +
292 (seq_group_metadata_list, scheduler_outputs,
293 allow_async_output_proc
294 ) = self.scheduler[virtual_engine].schedule()
@@ -290,8 +296,9 @@ class _AsyncLLMEngine(LLMEngine):
296 # If current scheduler iteration has no async postprocessor,
297 # then we need first to drain the pending async postprocessor
298 # before moving forward
293 - if not allow_async_output_proc and len(self.output_queue) > 0:
294 - self._process_model_outputs(is_async=True)
299 + if not allow_async_output_proc and len(ctx.output_queue) > 0:
300 + self._process_model_outputs(virtual_engine=virtual_engine,
301 + is_async=True)
302
303 if (self.scheduler_config.is_multi_step
304 and scheduler_outputs.num_lookahead_slots > 0):
@@ -332,8 +339,8 @@ class _AsyncLLMEngine(LLMEngine):
339 last_sampled_token_ids=last_sampled_token_ids)
340
341 if allow_async_output_proc:
335 - execute_model_req.output_proc_callback_fn = \
336 - self._process_model_outputs
342 + execute_model_req.async_callback = self.async_callback[
343 + virtual_engine]
344
345 # Execute the model.
346 output = await self.model_executor.execute_model_async(
@@ -343,9 +350,10 @@ class _AsyncLLMEngine(LLMEngine):
350 if self.scheduler_config.is_multi_step:
351 self._update_cached_scheduler_output(virtual_engine, output)
352 else:
346 - if len(self.output_queue) > 0:
353 + if len(ctx.output_queue) > 0:
354 assert not self.scheduler_config.is_multi_step
348 - self._process_model_outputs(is_async=True)
355 + self._process_model_outputs(virtual_engine=virtual_engine,
356 + is_async=True)
357 output = []
358
359 # Finish the current step for all the sequence groups.
@@ -360,7 +368,7 @@ class _AsyncLLMEngine(LLMEngine):
368 virtual_engine] = SchedulerOutputState()
369
370 # Cache results in engine
363 - self.output_queue.append(
371 + ctx.output_queue.append(
372 (output, seq_group_metadata_list, scheduler_outputs))
373
374 if output and allow_async_output_proc:
@@ -372,7 +380,8 @@ class _AsyncLLMEngine(LLMEngine):
380 scheduler_outputs.scheduled_seq_groups)
381
382 if not allow_async_output_proc:
375 - self._process_model_outputs(is_async=False)
383 + self._process_model_outputs(virtual_engine=virtual_engine,
384 + is_async=False)
385
386 # Log stats.
387 self.do_log_stats(scheduler_outputs, output)
@@ -381,9 +390,17 @@ class _AsyncLLMEngine(LLMEngine):
390 self.do_tracing(scheduler_outputs)
391
392 else:
384 - self.request_outputs = []
393 + ctx.request_outputs = []
394 +
395 + if not self.has_unfinished_requests():
396 + # Drain async postprocessor (if exists)
397 + if len(ctx.output_queue) > 0:
398 + assert not self.scheduler_config.is_multi_step
399 + self._process_model_outputs(virtual_engine=virtual_engine,
400 + is_async=True)
401 + assert len(ctx.output_queue) == 0
402
386 - return self.request_outputs
403 + return ctx.request_outputs
404
405 async def stop_remote_worker_execution_loop_async(self) -> None:
406 """Stop the remote worker execution loop."""
vllm/engine/llm_engine.py
+79 -42
@@ -1,7 +1,8 @@
1 +import functools
2 import time
3 from collections import deque
4 from contextlib import contextmanager
4 -from dataclasses import dataclass
5 +from dataclasses import dataclass, field
6 from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
7 Mapping, Optional)
8 from typing import Sequence as GenericSequence
@@ -88,6 +89,17 @@ class SchedulerOutputState:
89 last_output: Optional[SamplerOutput] = None
90
91
92 +@dataclass
93 +class SchedulerContext:
94 + output_queue: Deque[Tuple[List[SamplerOutput], List[SequenceGroupMetadata],
95 + SchedulerOutputs]] = field(
96 + default_factory=lambda: deque())
97 +
98 + request_outputs: List[Union[RequestOutput,
99 + EmbeddingRequestOutput]] = field(
100 + default_factory=lambda: [])
101 +
102 +
103 class LLMEngine:
104 """An LLM engine that receives requests and generates texts.
105
@@ -350,9 +362,11 @@ class LLMEngine:
362 Scheduler(
363 scheduler_config, cache_config, lora_config,
364 parallel_config.pipeline_parallel_size,
353 - self._process_model_outputs
365 + functools.partial(self._process_model_outputs,
366 + virtual_engine=v_id,
367 + is_async=True)
368 if model_config.use_async_output_proc else None)
355 - for _ in range(parallel_config.pipeline_parallel_size)
369 + for v_id in range(parallel_config.pipeline_parallel_size)
370 ]
371
372 # Metric Logging.
@@ -406,12 +420,17 @@ class LLMEngine:
420 for _ in range(self.parallel_config.pipeline_parallel_size)
421 ]
422
409 - # Async output processing pointers
410 - self.output_queue: Deque[Tuple[List[SamplerOutput],
411 - List[SequenceGroupMetadata],
412 - SchedulerOutputs]] = deque()
413 - self.request_outputs: List[Union[RequestOutput,
414 - EmbeddingRequestOutput]] = []
423 + self.scheduler_contexts = [
424 + SchedulerContext()
425 + for _ in range(self.parallel_config.pipeline_parallel_size)
426 + ]
427 +
428 + self.async_callback = [
429 + functools.partial(self._process_model_outputs,
430 + virtual_engine=v_id,
431 + is_async=True)
432 + for v_id in range(self.parallel_config.pipeline_parallel_size)
433 + ]
434
435 def _initialize_kv_caches(self) -> None:
436 """Initialize the KV cache in the worker(s).
@@ -1221,32 +1240,28 @@ class LLMEngine:
1240
1241 return
1242
1224 - def _process_model_outputs(self,
1225 - is_async: bool,
1226 - clear_outputs: bool = True) -> None:
1243 + def _process_model_outputs(self, virtual_engine: int,
1244 + is_async: bool) -> None:
1245 """Apply the model output to the sequences in the scheduled seq groups.
1246
1247 + virtual_engine: The engine id to operate on
1248 is_async: Indicates whether this postprocessor runs in
1249 parallel with the GPU forward pass and is processing
1250 tokens from the previous step. If this is true, then
1251 no tokens need to be appended since it is already done
1252 externally (before the next schedule() call)
1234 - clear_outputs: Sometimes existing outputs need to be combined
1235 - with outputs of this call. This happens for postprocessor
1236 - draining at the final stage (like when sequences are finished)
1253
1254 Returns RequestOutputs that can be returned to the client.
1255 """
1256 now = time.time()
1257
1242 - if clear_outputs:
1243 - self.request_outputs.clear()
1258 + ctx: SchedulerContext = self.scheduler_contexts[virtual_engine]
1259
1245 - if len(self.output_queue) == 0:
1260 + if len(ctx.output_queue) == 0:
1261 return None
1262
1263 (outputs, seq_group_metadata_list,
1249 - scheduler_outputs) = self.output_queue.popleft()
1264 + scheduler_outputs) = ctx.output_queue.popleft()
1265
1266 # Sanity check
1267 assert len(seq_group_metadata_list) == len(
@@ -1321,11 +1336,11 @@ class LLMEngine:
1336 if (seq_group.is_finished()
1337 if self.step_return_finished_only else True):
1338 request_output = RequestOutputFactory.create(seq_group)
1324 - self.request_outputs.append(request_output)
1339 + ctx.request_outputs.append(request_output)
1340
1341 for seq_group in scheduler_outputs.ignored_seq_groups:
1342 request_output = RequestOutputFactory.create(seq_group)
1328 - self.request_outputs.append(request_output)
1343 + ctx.request_outputs.append(request_output)
1344
1345 if is_async:
1346 # Log stats.
@@ -1421,29 +1436,43 @@ class LLMEngine:
1436 "Pipeline parallelism is only supported through AsyncLLMEngine "
1437 "as performance will be severely degraded otherwise.")
1438
1439 + # For llm_engine, there is no pipeline parallel support, so the engine
1440 + # used is always 0
1441 + virtual_engine = 0
1442 +
1443 # These are cached outputs from previous iterations. None if on first
1444 # iteration
1426 - cached_outputs = self.cached_scheduler_outputs[0]
1445 + cached_outputs = self.cached_scheduler_outputs[virtual_engine]
1446 seq_group_metadata_list = cached_outputs.seq_group_metadata_list
1447 scheduler_outputs = cached_outputs.scheduler_outputs
1448 allow_async_output_proc = cached_outputs.allow_async_output_proc
1449
1450 + ctx = self.scheduler_contexts[virtual_engine]
1451 +
1452 # Skip the scheduler if there are any remaining steps in the seq groups.
1453 # This ensures that the scheduler is only called again when the current
1454 # batch has completed.
1455 if not self._has_remaining_steps(seq_group_metadata_list):
1456 +
1457 + # Clear outputs on scheduler iteration start
1458 + ctx.request_outputs.clear()
1459 +
1460 + # Schedule iteration
1461 (seq_group_metadata_list, scheduler_outputs,
1436 - allow_async_output_proc) = self.scheduler[0].schedule()
1462 + allow_async_output_proc
1463 + ) = self.scheduler[virtual_engine].schedule()
1464
1438 - if not allow_async_output_proc and len(self.output_queue) > 0:
1439 - self._process_model_outputs(is_async=True)
1465 + # Maybe switch from async mode to sync mode
1466 + if not allow_async_output_proc and len(ctx.output_queue) > 0:
1467 + self._process_model_outputs(virtual_engine=virtual_engine,
1468 + is_async=True)
1469
1470 if (self.scheduler_config.is_multi_step
1471 and scheduler_outputs.num_lookahead_slots > 0):
1472 # cache the scheduler outputs for the next iteration if we have
1473 # lookahead slots
1474 self._cache_scheduler_outputs_for_multi_step(
1446 - 0, seq_group_metadata_list, scheduler_outputs,
1475 + virtual_engine, seq_group_metadata_list, scheduler_outputs,
1476 allow_async_output_proc)
1477
1478 assert seq_group_metadata_list is not None
@@ -1454,14 +1483,14 @@ class LLMEngine:
1483
1484 if not scheduler_outputs.is_empty():
1485 finished_requests_ids = self.scheduler[
1457 - 0].get_and_reset_finished_requests_ids()
1486 + virtual_engine].get_and_reset_finished_requests_ids()
1487
1488 # Check if we have a cached last_output from the previous iteration.
1489 # For supporting PP this is probably the best way to pass the
1490 # sampled_token_ids, as a separate broadcast over all the PP stages
1491 # will cause one virtual engine's microbatch to block the pipeline.
1492 last_sampled_token_ids = \
1464 - self._get_last_sampled_token_ids(0)
1493 + self._get_last_sampled_token_ids(virtual_engine)
1494
1495 execute_model_req = ExecuteModelRequest(
1496 seq_group_metadata_list=seq_group_metadata_list,
@@ -1476,20 +1505,24 @@ class LLMEngine:
1505 last_sampled_token_ids=last_sampled_token_ids)
1506
1507 if allow_async_output_proc:
1479 - execute_model_req.output_proc_callback_fn = \
1480 - self._process_model_outputs
1508 + execute_model_req.async_callback = self.async_callback[
1509 + virtual_engine]
1510
1511 output = self.model_executor.execute_model(
1512 execute_model_req=execute_model_req)
1513
1485 - # we need to do this here so that last step's sampled_token_ids can
1514 + # We need to do this here so that last step's sampled_token_ids can
1515 # be passed to the next iteration for PP.
1516 if self.scheduler_config.is_multi_step:
1488 - self._update_cached_scheduler_output(0, output)
1517 + self._update_cached_scheduler_output(virtual_engine, output)
1518 else:
1490 - if len(self.output_queue) > 0:
1519 + # Nothing scheduled => If there is pending async postprocessor,
1520 + # then finish it here.
1521 + if len(ctx.output_queue) > 0:
1522 assert not self.scheduler_config.is_multi_step
1492 - self._process_model_outputs(is_async=True)
1523 + self._process_model_outputs(virtual_engine=virtual_engine,
1524 + is_async=True)
1525 + # No outputs in this case
1526 output = []
1527
1528 # Finish the current step for all the sequence groups.
@@ -1504,7 +1537,7 @@ class LLMEngine:
1537
1538 # Add results to the output_queue
1539 # (for async or non-async postprocessing)
1507 - self.output_queue.append(
1540 + ctx.output_queue.append(
1541 (output, seq_group_metadata_list, scheduler_outputs))
1542
1543 if output and allow_async_output_proc:
@@ -1515,8 +1548,10 @@ class LLMEngine:
1548 output[0], seq_group_metadata_list,
1549 scheduler_outputs.scheduled_seq_groups)
1550
1551 + # Check if need to run the usual non-async path
1552 if not allow_async_output_proc:
1519 - self._process_model_outputs(is_async=False)
1553 + self._process_model_outputs(virtual_engine=virtual_engine,
1554 + is_async=False)
1555
1556 # Log stats.
1557 self.do_log_stats(scheduler_outputs, output)
@@ -1524,14 +1559,16 @@ class LLMEngine:
1559 # Tracing
1560 self.do_tracing(scheduler_outputs)
1561 else:
1527 - self.request_outputs = []
1562 + # Multi-step case
1563 + ctx.request_outputs = []
1564
1565 if not self.has_unfinished_requests():
1530 - # Drain async postprocessor
1531 - if len(self.output_queue) > 0:
1566 + # Drain async postprocessor (if exists)
1567 + if len(ctx.output_queue) > 0:
1568 assert not self.scheduler_config.is_multi_step
1533 - self._process_model_outputs(is_async=True, clear_outputs=False)
1534 - assert len(self.output_queue) == 0
1569 + self._process_model_outputs(virtual_engine=virtual_engine,
1570 + is_async=True)
1571 + assert len(ctx.output_queue) == 0
1572
1573 # Stop the execute model loop in parallel workers until there are
1574 # more requests to process. This avoids waiting indefinitely in
@@ -1540,7 +1577,7 @@ class LLMEngine:
1577 # queued control plane messages, such as add/remove lora adapters.
1578 self.model_executor.stop_remote_worker_execution_loop()
1579
1543 - return self.request_outputs
1580 + return ctx.request_outputs
1581
1582 def _has_remaining_steps(
1583 self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
vllm/sequence.py
+6 -3
@@ -811,6 +811,9 @@ class SequenceGroup:
811 self.is_single_seq = len(self.seqs) == 1
812
813 def is_finished(self) -> bool:
814 + if self.is_single_seq:
815 + return self.seqs[0].is_finished()
816 +
817 return all(seq.is_finished() for seq in self.seqs)
818
819 def is_prefill(self) -> bool:
@@ -1290,8 +1293,8 @@ class ExecuteModelRequest(
1293 finished_requests_ids: List[str] = msgspec.field(default_factory=list)
1294 # The last sampled token ids for multi step decoding.
1295 last_sampled_token_ids: Optional[torch.Tensor] = None
1293 - # Async postprocessor
1294 - output_proc_callback_fn: Optional[Callable] = None
1296 + # Async callback
1297 + async_callback: Optional[Callable] = None
1298
1299 @property
1300 def is_first_multi_step(self) -> bool:
@@ -1338,4 +1341,4 @@ class ExecuteModelRequest(
1341 finished_requests_ids=self.finished_requests_ids,
1342 last_sampled_token_ids=self.last_sampled_token_ids.clone()
1343 if self.last_sampled_token_ids is not None else None,
1341 - output_proc_callback_fn=self.output_proc_callback_fn)
1344 + async_callback=self.async_callback)
vllm/worker/model_runner.py
+3 -3
@@ -91,7 +91,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
91 request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
92 finished_requests_ids: Optional[List[str]] = None
93 virtual_engine: int = 0
94 - output_proc_callback_fn: Optional[Callable] = None
94 + async_callback: Optional[Callable] = None
95
96 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
97 tensor_dict = {
@@ -1457,8 +1457,8 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1457 if not self.is_driver_worker:
1458 return []
1459
1460 - if model_input.output_proc_callback_fn is not None:
1461 - model_input.output_proc_callback_fn(is_async=True)
1460 + if model_input.async_callback is not None:
1461 + model_input.async_callback()
1462
1463 # Sample the next token.
1464 output: SamplerOutput = self.model.sample(
vllm/worker/worker_base.py
+2 -3
@@ -263,11 +263,10 @@ class LocalOrDistributedWorkerBase(WorkerBase):
263 broadcast_data.update(kwargs)
264 broadcast_tensor_dict(broadcast_data, src=0)
265
266 - if execute_model_req.output_proc_callback_fn:
266 + if execute_model_req.async_callback:
267 model_input = dataclasses.replace( # type: ignore
268 model_input,
269 - output_proc_callback_fn=execute_model_req.
270 - output_proc_callback_fn)
269 + async_callback=execute_model_req.async_callback)
270
271 return model_input, worker_input, kwargs
272