A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Async_output_proc: Add virtual engine support (towards pipeline parallel) (#7911)
Alexander Matveev committed
Aug 28, 2024 at 03:02 UTC
f508e03e7f2d8aed897d8843e1ed1668e5c4ad7a
6 files changed
+122
-67
vllm/core/scheduler.py
+5
-6
@@ -302,7 +302,7 @@ class Scheduler:
302
cache_config: CacheConfig,
303
lora_config: Optional[LoRAConfig],
304
pipeline_parallel_size: int = 1,
305
- output_proc_callback_fn: Optional[Callable] = None,
305
+ output_proc_callback: Optional[Callable] = None,
306
) -> None:
307
self.scheduler_config = scheduler_config
308
self.cache_config = cache_config
@@ -376,8 +376,8 @@ class Scheduler:
376
# iterations. I.e. since the output processing is lagged one step,
377
# we cannot reuse the cached objects immediately when the schedule()
378
# is called again, but only when schedule() is called the second time.
379
- self.output_proc_callback_fn = output_proc_callback_fn
380
- self.use_async_output_proc = self.output_proc_callback_fn is not None
379
+ self.output_proc_callback = output_proc_callback
380
+ self.use_async_output_proc = self.output_proc_callback is not None
381
self.num_cache_iters = 2 if self.use_async_output_proc else 1
382
383
self.cache_id = 0
@@ -573,8 +573,8 @@ class Scheduler:
573
seq_group):
574
tmp = self.running
575
self.running = orig_running
576
- assert self.output_proc_callback_fn is not None
577
- self.output_proc_callback_fn(is_async=True)
576
+ assert self.output_proc_callback is not None
577
+ self.output_proc_callback()
578
self.running = tmp
579
580
while not self._can_append_slots(seq_group):
@@ -1091,7 +1091,6 @@ class Scheduler:
1091
no_beam_search = seq_group.sampling_params is None or (
1092
seq_group.sampling_params.best_of == 1
1093
and not seq_group.sampling_params.use_beam_search)
1094
-
1094
return no_beam_search
1095
1096
def schedule(
vllm/engine/async_llm_engine.py
+27
-10
@@ -279,10 +279,16 @@ class _AsyncLLMEngine(LLMEngine):
279
scheduler_outputs = cached_outputs.scheduler_outputs
280
allow_async_output_proc = cached_outputs.allow_async_output_proc
281
282
+ ctx = self.scheduler_contexts[virtual_engine]
283
+
284
# skip the scheduler if there are any remaining steps in the seq groups.
285
# This ensures that the scheduler is only called again when the current
286
# batch has completed.
287
if not self._has_remaining_steps(seq_group_metadata_list):
288
+
289
+ # Clear outputs on scheduler iteration start
290
+ ctx.request_outputs.clear()
291
+
292
(seq_group_metadata_list, scheduler_outputs,
293
allow_async_output_proc
294
) = self.scheduler[virtual_engine].schedule()
@@ -290,8 +296,9 @@ class _AsyncLLMEngine(LLMEngine):
296
# If current scheduler iteration has no async postprocessor,
297
# then we need first to drain the pending async postprocessor
298
# before moving forward
293
- if not allow_async_output_proc and len(self.output_queue) > 0:
294
- self._process_model_outputs(is_async=True)
299
+ if not allow_async_output_proc and len(ctx.output_queue) > 0:
300
+ self._process_model_outputs(virtual_engine=virtual_engine,
301
+ is_async=True)
302
303
if (self.scheduler_config.is_multi_step
304
and scheduler_outputs.num_lookahead_slots > 0):
@@ -332,8 +339,8 @@ class _AsyncLLMEngine(LLMEngine):
339
last_sampled_token_ids=last_sampled_token_ids)
340
341
if allow_async_output_proc:
335
- execute_model_req.output_proc_callback_fn = \
336
- self._process_model_outputs
342
+ execute_model_req.async_callback = self.async_callback[
343
+ virtual_engine]
344
345
# Execute the model.
346
output = await self.model_executor.execute_model_async(
@@ -343,9 +350,10 @@ class _AsyncLLMEngine(LLMEngine):
350
if self.scheduler_config.is_multi_step:
351
self._update_cached_scheduler_output(virtual_engine, output)
352
else:
346
- if len(self.output_queue) > 0:
353
+ if len(ctx.output_queue) > 0:
354
assert not self.scheduler_config.is_multi_step
348
- self._process_model_outputs(is_async=True)
355
+ self._process_model_outputs(virtual_engine=virtual_engine,
356
+ is_async=True)
357
output = []
358
359
# Finish the current step for all the sequence groups.
@@ -360,7 +368,7 @@ class _AsyncLLMEngine(LLMEngine):
368
virtual_engine] = SchedulerOutputState()
369
370
# Cache results in engine
363
- self.output_queue.append(
371
+ ctx.output_queue.append(
372
(output, seq_group_metadata_list, scheduler_outputs))
373
374
if output and allow_async_output_proc:
@@ -372,7 +380,8 @@ class _AsyncLLMEngine(LLMEngine):
380
scheduler_outputs.scheduled_seq_groups)
381
382
if not allow_async_output_proc:
375
- self._process_model_outputs(is_async=False)
383
+ self._process_model_outputs(virtual_engine=virtual_engine,
384
+ is_async=False)
385
386
# Log stats.
387
self.do_log_stats(scheduler_outputs, output)
@@ -381,9 +390,17 @@ class _AsyncLLMEngine(LLMEngine):
390
self.do_tracing(scheduler_outputs)
391
392
else:
384
- self.request_outputs = []
393
+ ctx.request_outputs = []
394
+
395
+ if not self.has_unfinished_requests():
396
+ # Drain async postprocessor (if exists)
397
+ if len(ctx.output_queue) > 0:
398
+ assert not self.scheduler_config.is_multi_step
399
+ self._process_model_outputs(virtual_engine=virtual_engine,
400
+ is_async=True)
401
+ assert len(ctx.output_queue) == 0
402
386
- return self.request_outputs
403
+ return ctx.request_outputs
404
405
async def stop_remote_worker_execution_loop_async(self) -> None:
406
"""Stop the remote worker execution loop."""
vllm/engine/llm_engine.py
+79
-42
@@ -1,7 +1,8 @@
1
+import functools
2
import time
3
from collections import deque
4
from contextlib import contextmanager
4
-from dataclasses import dataclass
5
+from dataclasses import dataclass, field
6
from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
7
Mapping, Optional)
8
from typing import Sequence as GenericSequence
@@ -88,6 +89,17 @@ class SchedulerOutputState:
89
last_output: Optional[SamplerOutput] = None
90
91
92
+@dataclass
93
+class SchedulerContext:
94
+ output_queue: Deque[Tuple[List[SamplerOutput], List[SequenceGroupMetadata],
95
+ SchedulerOutputs]] = field(
96
+ default_factory=lambda: deque())
97
+
98
+ request_outputs: List[Union[RequestOutput,
99
+ EmbeddingRequestOutput]] = field(
100
+ default_factory=lambda: [])
101
+
102
+
103
class LLMEngine:
104
"""An LLM engine that receives requests and generates texts.
105
@@ -350,9 +362,11 @@ class LLMEngine:
362
Scheduler(
363
scheduler_config, cache_config, lora_config,
364
parallel_config.pipeline_parallel_size,
353
- self._process_model_outputs
365
+ functools.partial(self._process_model_outputs,
366
+ virtual_engine=v_id,
367
+ is_async=True)
368
if model_config.use_async_output_proc else None)
355
- for _ in range(parallel_config.pipeline_parallel_size)
369
+ for v_id in range(parallel_config.pipeline_parallel_size)
370
]
371
372
# Metric Logging.
@@ -406,12 +420,17 @@ class LLMEngine:
420
for _ in range(self.parallel_config.pipeline_parallel_size)
421
]
422
409
- # Async output processing pointers
410
- self.output_queue: Deque[Tuple[List[SamplerOutput],
411
- List[SequenceGroupMetadata],
412
- SchedulerOutputs]] = deque()
413
- self.request_outputs: List[Union[RequestOutput,
414
- EmbeddingRequestOutput]] = []
423
+ self.scheduler_contexts = [
424
+ SchedulerContext()
425
+ for _ in range(self.parallel_config.pipeline_parallel_size)
426
+ ]
427
+
428
+ self.async_callback = [
429
+ functools.partial(self._process_model_outputs,
430
+ virtual_engine=v_id,
431
+ is_async=True)
432
+ for v_id in range(self.parallel_config.pipeline_parallel_size)
433
+ ]
434
435
def _initialize_kv_caches(self) -> None:
436
"""Initialize the KV cache in the worker(s).
@@ -1221,32 +1240,28 @@ class LLMEngine:
1240
1241
return
1242
1224
- def _process_model_outputs(self,
1225
- is_async: bool,
1226
- clear_outputs: bool = True) -> None:
1243
+ def _process_model_outputs(self, virtual_engine: int,
1244
+ is_async: bool) -> None:
1245
"""Apply the model output to the sequences in the scheduled seq groups.
1246
1247
+ virtual_engine: The engine id to operate on
1248
is_async: Indicates whether this postprocessor runs in
1249
parallel with the GPU forward pass and is processing
1250
tokens from the previous step. If this is true, then
1251
no tokens need to be appended since it is already done
1252
externally (before the next schedule() call)
1234
- clear_outputs: Sometimes existing outputs need to be combined
1235
- with outputs of this call. This happens for postprocessor
1236
- draining at the final stage (like when sequences are finished)
1253
1254
Returns RequestOutputs that can be returned to the client.
1255
"""
1256
now = time.time()
1257
1242
- if clear_outputs:
1243
- self.request_outputs.clear()
1258
+ ctx: SchedulerContext = self.scheduler_contexts[virtual_engine]
1259
1245
- if len(self.output_queue) == 0:
1260
+ if len(ctx.output_queue) == 0:
1261
return None
1262
1263
(outputs, seq_group_metadata_list,
1249
- scheduler_outputs) = self.output_queue.popleft()
1264
+ scheduler_outputs) = ctx.output_queue.popleft()
1265
1266
# Sanity check
1267
assert len(seq_group_metadata_list) == len(
@@ -1321,11 +1336,11 @@ class LLMEngine:
1336
if (seq_group.is_finished()
1337
if self.step_return_finished_only else True):
1338
request_output = RequestOutputFactory.create(seq_group)
1324
- self.request_outputs.append(request_output)
1339
+ ctx.request_outputs.append(request_output)
1340
1341
for seq_group in scheduler_outputs.ignored_seq_groups:
1342
request_output = RequestOutputFactory.create(seq_group)
1328
- self.request_outputs.append(request_output)
1343
+ ctx.request_outputs.append(request_output)
1344
1345
if is_async:
1346
# Log stats.
@@ -1421,29 +1436,43 @@ class LLMEngine:
1436
"Pipeline parallelism is only supported through AsyncLLMEngine "
1437
"as performance will be severely degraded otherwise.")
1438
1439
+ # For llm_engine, there is no pipeline parallel support, so the engine
1440
+ # used is always 0
1441
+ virtual_engine = 0
1442
+
1443
# These are cached outputs from previous iterations. None if on first
1444
# iteration
1426
- cached_outputs = self.cached_scheduler_outputs[0]
1445
+ cached_outputs = self.cached_scheduler_outputs[virtual_engine]
1446
seq_group_metadata_list = cached_outputs.seq_group_metadata_list
1447
scheduler_outputs = cached_outputs.scheduler_outputs
1448
allow_async_output_proc = cached_outputs.allow_async_output_proc
1449
1450
+ ctx = self.scheduler_contexts[virtual_engine]
1451
+
1452
# Skip the scheduler if there are any remaining steps in the seq groups.
1453
# This ensures that the scheduler is only called again when the current
1454
# batch has completed.
1455
if not self._has_remaining_steps(seq_group_metadata_list):
1456
+
1457
+ # Clear outputs on scheduler iteration start
1458
+ ctx.request_outputs.clear()
1459
+
1460
+ # Schedule iteration
1461
(seq_group_metadata_list, scheduler_outputs,
1436
- allow_async_output_proc) = self.scheduler[0].schedule()
1462
+ allow_async_output_proc
1463
+ ) = self.scheduler[virtual_engine].schedule()
1464
1438
- if not allow_async_output_proc and len(self.output_queue) > 0:
1439
- self._process_model_outputs(is_async=True)
1465
+ # Maybe switch from async mode to sync mode
1466
+ if not allow_async_output_proc and len(ctx.output_queue) > 0:
1467
+ self._process_model_outputs(virtual_engine=virtual_engine,
1468
+ is_async=True)
1469
1470
if (self.scheduler_config.is_multi_step
1471
and scheduler_outputs.num_lookahead_slots > 0):
1472
# cache the scheduler outputs for the next iteration if we have
1473
# lookahead slots
1474
self._cache_scheduler_outputs_for_multi_step(
1446
- 0, seq_group_metadata_list, scheduler_outputs,
1475
+ virtual_engine, seq_group_metadata_list, scheduler_outputs,
1476
allow_async_output_proc)
1477
1478
assert seq_group_metadata_list is not None
@@ -1454,14 +1483,14 @@ class LLMEngine:
1483
1484
if not scheduler_outputs.is_empty():
1485
finished_requests_ids = self.scheduler[
1457
- 0].get_and_reset_finished_requests_ids()
1486
+ virtual_engine].get_and_reset_finished_requests_ids()
1487
1488
# Check if we have a cached last_output from the previous iteration.
1489
# For supporting PP this is probably the best way to pass the
1490
# sampled_token_ids, as a separate broadcast over all the PP stages
1491
# will cause one virtual engine's microbatch to block the pipeline.
1492
last_sampled_token_ids = \
1464
- self._get_last_sampled_token_ids(0)
1493
+ self._get_last_sampled_token_ids(virtual_engine)
1494
1495
execute_model_req = ExecuteModelRequest(
1496
seq_group_metadata_list=seq_group_metadata_list,
@@ -1476,20 +1505,24 @@ class LLMEngine:
1505
last_sampled_token_ids=last_sampled_token_ids)
1506
1507
if allow_async_output_proc:
1479
- execute_model_req.output_proc_callback_fn = \
1480
- self._process_model_outputs
1508
+ execute_model_req.async_callback = self.async_callback[
1509
+ virtual_engine]
1510
1511
output = self.model_executor.execute_model(
1512
execute_model_req=execute_model_req)
1513
1485
- # we need to do this here so that last step's sampled_token_ids can
1514
+ # We need to do this here so that last step's sampled_token_ids can
1515
# be passed to the next iteration for PP.
1516
if self.scheduler_config.is_multi_step:
1488
- self._update_cached_scheduler_output(0, output)
1517
+ self._update_cached_scheduler_output(virtual_engine, output)
1518
else:
1490
- if len(self.output_queue) > 0:
1519
+ # Nothing scheduled => If there is pending async postprocessor,
1520
+ # then finish it here.
1521
+ if len(ctx.output_queue) > 0:
1522
assert not self.scheduler_config.is_multi_step
1492
- self._process_model_outputs(is_async=True)
1523
+ self._process_model_outputs(virtual_engine=virtual_engine,
1524
+ is_async=True)
1525
+ # No outputs in this case
1526
output = []
1527
1528
# Finish the current step for all the sequence groups.
@@ -1504,7 +1537,7 @@ class LLMEngine:
1537
1538
# Add results to the output_queue
1539
# (for async or non-async postprocessing)
1507
- self.output_queue.append(
1540
+ ctx.output_queue.append(
1541
(output, seq_group_metadata_list, scheduler_outputs))
1542
1543
if output and allow_async_output_proc:
@@ -1515,8 +1548,10 @@ class LLMEngine:
1548
output[0], seq_group_metadata_list,
1549
scheduler_outputs.scheduled_seq_groups)
1550
1551
+ # Check if need to run the usual non-async path
1552
if not allow_async_output_proc:
1519
- self._process_model_outputs(is_async=False)
1553
+ self._process_model_outputs(virtual_engine=virtual_engine,
1554
+ is_async=False)
1555
1556
# Log stats.
1557
self.do_log_stats(scheduler_outputs, output)
@@ -1524,14 +1559,16 @@ class LLMEngine:
1559
# Tracing
1560
self.do_tracing(scheduler_outputs)
1561
else:
1527
- self.request_outputs = []
1562
+ # Multi-step case
1563
+ ctx.request_outputs = []
1564
1565
if not self.has_unfinished_requests():
1530
- # Drain async postprocessor
1531
- if len(self.output_queue) > 0:
1566
+ # Drain async postprocessor (if exists)
1567
+ if len(ctx.output_queue) > 0:
1568
assert not self.scheduler_config.is_multi_step
1533
- self._process_model_outputs(is_async=True, clear_outputs=False)
1534
- assert len(self.output_queue) == 0
1569
+ self._process_model_outputs(virtual_engine=virtual_engine,
1570
+ is_async=True)
1571
+ assert len(ctx.output_queue) == 0
1572
1573
# Stop the execute model loop in parallel workers until there are
1574
# more requests to process. This avoids waiting indefinitely in
@@ -1540,7 +1577,7 @@ class LLMEngine:
1577
# queued control plane messages, such as add/remove lora adapters.
1578
self.model_executor.stop_remote_worker_execution_loop()
1579
1543
- return self.request_outputs
1580
+ return ctx.request_outputs
1581
1582
def _has_remaining_steps(
1583
self, seq_group_metadata_list: Optional[List[SequenceGroupMetadata]]
vllm/sequence.py
+6
-3
@@ -811,6 +811,9 @@ class SequenceGroup:
811
self.is_single_seq = len(self.seqs) == 1
812
813
def is_finished(self) -> bool:
814
+ if self.is_single_seq:
815
+ return self.seqs[0].is_finished()
816
+
817
return all(seq.is_finished() for seq in self.seqs)
818
819
def is_prefill(self) -> bool:
@@ -1290,8 +1293,8 @@ class ExecuteModelRequest(
1293
finished_requests_ids: List[str] = msgspec.field(default_factory=list)
1294
# The last sampled token ids for multi step decoding.
1295
last_sampled_token_ids: Optional[torch.Tensor] = None
1293
- # Async postprocessor
1294
- output_proc_callback_fn: Optional[Callable] = None
1296
+ # Async callback
1297
+ async_callback: Optional[Callable] = None
1298
1299
@property
1300
def is_first_multi_step(self) -> bool:
@@ -1338,4 +1341,4 @@ class ExecuteModelRequest(
1341
finished_requests_ids=self.finished_requests_ids,
1342
last_sampled_token_ids=self.last_sampled_token_ids.clone()
1343
if self.last_sampled_token_ids is not None else None,
1341
- output_proc_callback_fn=self.output_proc_callback_fn)
1344
+ async_callback=self.async_callback)
vllm/worker/model_runner.py
+3
-3
@@ -91,7 +91,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
91
request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
92
finished_requests_ids: Optional[List[str]] = None
93
virtual_engine: int = 0
94
- output_proc_callback_fn: Optional[Callable] = None
94
+ async_callback: Optional[Callable] = None
95
96
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
97
tensor_dict = {
@@ -1457,8 +1457,8 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1457
if not self.is_driver_worker:
1458
return []
1459
1460
- if model_input.output_proc_callback_fn is not None:
1461
- model_input.output_proc_callback_fn(is_async=True)
1460
+ if model_input.async_callback is not None:
1461
+ model_input.async_callback()
1462
1463
# Sample the next token.
1464
output: SamplerOutput = self.model.sample(
vllm/worker/worker_base.py
+2
-3
@@ -263,11 +263,10 @@ class LocalOrDistributedWorkerBase(WorkerBase):
263
broadcast_data.update(kwargs)
264
broadcast_tensor_dict(broadcast_data, src=0)
265
266
- if execute_model_req.output_proc_callback_fn:
266
+ if execute_model_req.async_callback:
267
model_input = dataclasses.replace( # type: ignore
268
model_input,
269
- output_proc_callback_fn=execute_model_req.
270
- output_proc_callback_fn)
269
+ async_callback=execute_model_req.async_callback)
270
271
return model_input, worker_input, kwargs
272