@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d646d08a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core] Optimize Async + Multi-step (#8050)

Alexander Matveev committed Sep 3, 2024 at 14:50 UTC 6d646d08a2e0e73e83e313a5ae470c1f9e4f200e
8 files changed +326 -248
tests/multi_step/test_correctness_async_llm.py
+2 -2
@@ -103,13 +103,13 @@ async def test_multi_step(
103 model,
104 server_args + distributed_args,
105 num_logprobs,
106 - max_wait_seconds=3 * 240)
106 + max_wait_seconds=5 * 240)
107 test_completions = await completions_with_server_args(
108 prompts,
109 model,
110 ms_server_args + distributed_args,
111 num_logprobs,
112 - max_wait_seconds=3 * 240)
112 + max_wait_seconds=5 * 240)
113
114 # Assert multi-step scheduling produces identical tokens
115 # to single-step scheduling.
vllm/engine/async_llm_engine.py
+54 -55
@@ -280,40 +280,27 @@ class _AsyncLLMEngine(LLMEngine):
280 scheduler_outputs = cached_outputs.scheduler_outputs
281 allow_async_output_proc = cached_outputs.allow_async_output_proc
282
283 - # Detect async + multi-step
284 - use_async_and_multi_step = (self.scheduler_config.is_multi_step
285 - and allow_async_output_proc)
286 -
283 ctx = self.scheduler_contexts[virtual_engine]
284
285 + # Clear outputs for each new scheduler iteration
286 + ctx.request_outputs.clear()
287 +
288 # skip the scheduler if there are any remaining steps in the seq groups.
289 # This ensures that the scheduler is only called again when the current
290 # batch has completed.
291 if not self._has_remaining_steps(seq_group_metadata_list):
292
294 - # Clear outputs on scheduler iteration start
295 - ctx.request_outputs.clear()
296 -
293 # Schedule iteration
294 (seq_group_metadata_list, scheduler_outputs,
295 allow_async_output_proc
296 ) = self.scheduler[virtual_engine].schedule()
297
302 - # Detect async + multi-step
303 - use_async_and_multi_step = (self.scheduler_config.is_multi_step
304 - and allow_async_output_proc)
298 + ctx.seq_group_metadata_list = seq_group_metadata_list
299 + ctx.scheduler_outputs = scheduler_outputs
300
301 # Maybe switch from async mode to sync mode
302 if not allow_async_output_proc and len(ctx.output_queue) > 0:
308 - self._process_model_outputs(virtual_engine=virtual_engine,
309 - is_async=True)
310 -
311 - # For async + multi-step, init the queue
312 - if use_async_and_multi_step:
313 - assert len(ctx.output_queue) == 0
314 - assert seq_group_metadata_list is not None
315 - ctx.output_queue.append(
316 - (None, seq_group_metadata_list, scheduler_outputs))
303 + self._process_model_outputs(ctx=ctx)
304
305 if (self.scheduler_config.is_multi_step
306 and scheduler_outputs.num_lookahead_slots > 0):
@@ -351,26 +338,20 @@ class _AsyncLLMEngine(LLMEngine):
338 last_sampled_token_ids=last_sampled_token_ids)
339
340 if allow_async_output_proc:
354 - async_callback = self.async_callback_multi_step[
355 - virtual_engine] if use_async_and_multi_step \
356 - else self.async_callback[virtual_engine]
357 -
358 - execute_model_req.async_callback = async_callback
359 - execute_model_req.use_async_and_multi_step = \
360 - use_async_and_multi_step
341 + execute_model_req.async_callback = self.async_callbacks[
342 + virtual_engine]
343
344 # Execute the model.
345 output = await self.model_executor.execute_model_async(
346 execute_model_req)
347 +
348 # we need to do this here so that last step's sampled_token_ids can
349 # be passed to the next iteration for PP.
350 if self.scheduler_config.is_multi_step:
351 self._update_cached_scheduler_output(virtual_engine, output)
352 else:
370 - if not use_async_and_multi_step and len(ctx.output_queue) > 0:
371 - assert not self.scheduler_config.is_multi_step
372 - self._process_model_outputs(virtual_engine=virtual_engine,
373 - is_async=True)
353 + if len(ctx.output_queue) > 0:
354 + self._process_model_outputs(ctx=ctx)
355 output = []
356
357 # Finish the current step for all the sequence groups.
@@ -384,24 +365,22 @@ class _AsyncLLMEngine(LLMEngine):
365 self.cached_scheduler_outputs[
366 virtual_engine] = SchedulerOutputState()
367
387 - if use_async_and_multi_step:
388 - # For async + multi-step, clear the queue
389 - ctx.output_queue.clear()
390 - else:
391 - ctx.output_queue.append(
392 - (output, seq_group_metadata_list, scheduler_outputs))
368 + is_async = allow_async_output_proc
369 + is_last_step = True
370 + ctx.output_queue.append(
371 + (output, seq_group_metadata_list, scheduler_outputs, is_async,
372 + is_last_step))
373
394 - if output and allow_async_output_proc:
395 - assert len(
396 - output
397 - ) == 1, "Multi step decoding does not work with async output processing." # noqa: E501
398 - self._advance_to_next_step(
399 - output[0], seq_group_metadata_list,
400 - scheduler_outputs.scheduled_seq_groups)
374 + if output and allow_async_output_proc:
375 + assert len(
376 + output
377 + ) == 1, "Async postprocessor expects only a single output set"
378 + self._advance_to_next_step(
379 + output[0], seq_group_metadata_list,
380 + scheduler_outputs.scheduled_seq_groups)
381
382 if not allow_async_output_proc:
403 - self._process_model_outputs(virtual_engine=virtual_engine,
404 - is_async=False)
383 + self._process_model_outputs(ctx=ctx)
384
385 # Log stats.
386 self.do_log_stats(scheduler_outputs, output)
@@ -411,17 +390,12 @@ class _AsyncLLMEngine(LLMEngine):
390
391 else:
392 # Multi-step case
414 - if use_async_and_multi_step:
415 - return []
416 - else:
417 - ctx.request_outputs = []
393 + return ctx.request_outputs
394
395 if not self.has_unfinished_requests():
396 # Drain async postprocessor (if exists)
397 if len(ctx.output_queue) > 0:
422 - assert not self.scheduler_config.is_multi_step
423 - self._process_model_outputs(virtual_engine=virtual_engine,
424 - is_async=True)
398 + self._process_model_outputs(ctx=ctx)
399 assert len(ctx.output_queue) == 0
400
401 return ctx.request_outputs
@@ -640,6 +614,17 @@ class AsyncLLMEngine:
614 self.log_requests = log_requests
615 self.engine = self._init_engine(*args, **kwargs)
616
617 + # This ensures quick processing of request outputs
618 + # so the append to asyncio queues is not delayed,
619 + # especially for multi-step.
620 + #
621 + # TODO: Currently, disabled for engine_use_ray, ask
622 + # Cody/Will/Woosuk about this case.
623 + self.use_process_request_outputs_callback = not self.engine_use_ray
624 + if self.use_process_request_outputs_callback:
625 + self.engine.process_request_outputs_callback = \
626 + self.process_request_outputs
627 +
628 if self.engine_use_ray:
629 print_warning_once(
630 "DEPRECATED. `--engine-use-ray` is deprecated and will "
@@ -883,13 +868,27 @@ class AsyncLLMEngine:
868 request_outputs = await self.engine.step_async(virtual_engine)
869
870 # Put the outputs into the corresponding streams.
886 - finished = True
871 + # If used as a callback, then already invoked inside
872 + # LLMEngine's _process_model_outputs
873 + if not self.use_process_request_outputs_callback:
874 + all_finished = self.process_request_outputs(request_outputs)
875 + else:
876 + # For callback case, we only need to detect when all
877 + # requests are finished
878 + all_finished = all(request_output.finished
879 + for request_output in request_outputs)
880 +
881 + return not all_finished
882 +
883 + def process_request_outputs(self, request_outputs) -> bool:
884 + # Put the outputs into the corresponding streams.
885 + all_finished = True
886 for request_output in request_outputs:
887 self._request_tracker.process_request_output(
888 request_output, verbose=self.log_requests)
890 - finished = finished and request_output.finished
889 + all_finished = all_finished and request_output.finished
890
892 - return not finished
891 + return all_finished
892
893 async def _engine_abort(self, request_ids: Iterable[str]):
894 if self.engine_use_ray:
vllm/engine/llm_engine.py
+97 -125
@@ -93,13 +93,14 @@ class SchedulerOutputState:
93 @dataclass
94 class SchedulerContext:
95 output_queue: Deque[Tuple[Optional[List[SamplerOutput]],
96 - List[SequenceGroupMetadata],
97 - SchedulerOutputs]] = field(
98 - default_factory=lambda: deque())
99 -
96 + List[SequenceGroupMetadata], SchedulerOutputs,
97 + bool,
98 + bool]] = field(default_factory=lambda: deque())
99 request_outputs: List[Union[RequestOutput,
100 EmbeddingRequestOutput]] = field(
101 default_factory=lambda: [])
102 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
103 + scheduler_outputs: Optional[SchedulerOutputs] = None
104
105
106 class LLMEngine:
@@ -357,6 +358,26 @@ class LLMEngine:
358 # different process.
359 self.tokenizer.ping()
360
361 + self.cached_scheduler_outputs = [
362 + SchedulerOutputState()
363 + for _ in range(self.parallel_config.pipeline_parallel_size)
364 + ]
365 +
366 + self.scheduler_contexts = [
367 + SchedulerContext()
368 + for _ in range(self.parallel_config.pipeline_parallel_size)
369 + ]
370 +
371 + self.async_callbacks = [
372 + functools.partial(self._process_model_outputs,
373 + ctx=self.scheduler_contexts[v_id])
374 + for v_id in range(self.parallel_config.pipeline_parallel_size)
375 + ]
376 +
377 + # Currently used by AsyncLLMEngine to ensure quick append
378 + # of request outputs to asyncio queues
379 + self.process_request_outputs_callback = None
380 +
381 # Create the scheduler.
382 # NOTE: the cache_config here have been updated with the numbers of
383 # GPU and CPU blocks, which are profiled in the distributed executor.
@@ -364,9 +385,7 @@ class LLMEngine:
385 Scheduler(
386 scheduler_config, cache_config, lora_config,
387 parallel_config.pipeline_parallel_size,
367 - functools.partial(self._process_model_outputs,
368 - virtual_engine=v_id,
369 - is_async=True)
388 + self.async_callbacks[v_id]
389 if model_config.use_async_output_proc else None)
390 for v_id in range(parallel_config.pipeline_parallel_size)
391 ]
@@ -417,30 +436,6 @@ class LLMEngine:
436 ),
437 ))
438
420 - self.cached_scheduler_outputs = [
421 - SchedulerOutputState()
422 - for _ in range(self.parallel_config.pipeline_parallel_size)
423 - ]
424 -
425 - self.scheduler_contexts = [
426 - SchedulerContext()
427 - for _ in range(self.parallel_config.pipeline_parallel_size)
428 - ]
429 -
430 - self.async_callback = [
431 - functools.partial(self._process_model_outputs,
432 - virtual_engine=v_id,
433 - is_async=True)
434 - for v_id in range(self.parallel_config.pipeline_parallel_size)
435 - ]
436 -
437 - self.async_callback_multi_step = [
438 - functools.partial(self._process_model_outputs,
439 - virtual_engine=v_id,
440 - is_async=False)
441 - for v_id in range(self.parallel_config.pipeline_parallel_size)
442 - ]
443 -
439 def _initialize_kv_caches(self) -> None:
440 """Initialize the KV cache in the worker(s).
441
@@ -1249,11 +1244,7 @@ class LLMEngine:
1244
1245 return
1246
1252 - def _process_model_outputs(self,
1253 - virtual_engine: int,
1254 - is_async: bool,
1255 - sampler_output: Optional[SamplerOutput] = None,
1256 - is_last_output: bool = False) -> None:
1247 + def _process_model_outputs(self, ctx: SchedulerContext) -> None:
1248 """Apply the model output to the sequences in the scheduled seq groups.
1249
1250 virtual_engine: The engine id to operate on
@@ -1273,24 +1264,12 @@ class LLMEngine:
1264 """
1265 now = time.time()
1266
1276 - is_multi_step = sampler_output is not None
1277 -
1278 - ctx: SchedulerContext = self.scheduler_contexts[virtual_engine]
1279 -
1267 if len(ctx.output_queue) == 0:
1268 return None
1269
1283 - if is_multi_step:
1284 - # Async + multi-step case
1285 - (outputs, seq_group_metadata_list,
1286 - scheduler_outputs) = ctx.output_queue[0]
1287 - assert outputs is None
1288 - outputs = [sampler_output]
1289 - else:
1290 - # Async standard case
1291 - (outputs, seq_group_metadata_list,
1292 - scheduler_outputs) = ctx.output_queue.popleft()
1293 -
1270 + # Get pending async postprocessor
1271 + (outputs, seq_group_metadata_list, scheduler_outputs, is_async,
1272 + is_last_step) = ctx.output_queue.popleft()
1273 assert outputs is not None
1274
1275 # Sanity check
@@ -1306,6 +1285,7 @@ class LLMEngine:
1285 outputs_by_sequence_group = outputs
1286
1287 finished_before: List[int] = []
1288 + finished_now: List[int] = []
1289 for i, seq_group_meta in enumerate(seq_group_metadata_list):
1290 scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1291
@@ -1343,26 +1323,44 @@ class LLMEngine:
1323
1324 if self.model_config.embedding_mode:
1325 self._process_sequence_group_outputs(seq_group, output)
1346 - continue
1326 + else:
1327 + self.output_processor.process_prompt_logprob(seq_group, output)
1328 + if seq_group_meta.do_sample:
1329 + self.output_processor.process_outputs(
1330 + seq_group, output, is_async)
1331
1348 - self.output_processor.process_prompt_logprob(seq_group, output)
1349 - if seq_group_meta.do_sample:
1350 - self.output_processor.process_outputs(seq_group, output,
1351 - is_async)
1332 + if seq_group.is_finished():
1333 + finished_now.append(i)
1334
1353 - # For async + multi-step, free finished seqs and create outputs
1354 - # only on the final step.
1355 - if is_multi_step and not is_last_output:
1356 - return
1335 + # Generate outputs for the requests that finished this iteration
1336 + for i in finished_now:
1337 + scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1338
1358 - for scheduler in self.scheduler:
1359 - scheduler.free_finished_seq_groups()
1339 + seq_group = scheduled_seq_group.seq_group
1340 + seq_group.maybe_set_first_token_time(now)
1341 + request_output = RequestOutputFactory.create(seq_group)
1342 + ctx.request_outputs.append(request_output)
1343
1361 - # Create the outputs.
1362 - for i, _ in enumerate(seq_group_metadata_list):
1363 - scheduled_seq_group = scheduler_outputs.scheduled_seq_groups[i]
1344 + # Free currently finished requests
1345 + if finished_now:
1346 + for scheduler in self.scheduler:
1347 + scheduler.free_finished_seq_groups()
1348 +
1349 + # For multi-step, do not create outputs each iteration
1350 + if not is_last_step:
1351 + # Immediately process request outputs here (if callback is given)
1352 + if (finished_now
1353 + and self.process_request_outputs_callback is not None):
1354 + self.process_request_outputs_callback(ctx.request_outputs)
1355 + return
1356 +
1357 + # Create the outputs
1358 + # Note: scheduled_seq_groups and seq_group_metadata_list
1359 + # must match with the indices
1360 + for i, scheduled_seq_group in enumerate(
1361 + scheduler_outputs.scheduled_seq_groups):
1362
1365 - if not is_multi_step and i in finished_before:
1363 + if i in finished_before or i in finished_now:
1364 continue # Avoids double processing
1365
1366 seq_group = scheduled_seq_group.seq_group
@@ -1376,11 +1374,15 @@ class LLMEngine:
1374 request_output = RequestOutputFactory.create(seq_group)
1375 ctx.request_outputs.append(request_output)
1376
1379 - # For async + multi-step, do stats only on the last output.
1380 - # Otherwise, do stats if the execution is async
1381 - do_stats = is_multi_step or is_async
1377 + # Immediately process request outputs here (if callback is given)
1378 + if (ctx.request_outputs
1379 + and self.process_request_outputs_callback is not None):
1380 + self.process_request_outputs_callback(ctx.request_outputs)
1381
1383 - if do_stats:
1382 + # For async case, we need to record the stats here.
1383 + # For non-async case, the stats are done in the
1384 + # LLMEngine/AsyncLLMEngine directly
1385 + if is_async:
1386 # Log stats.
1387 self.do_log_stats(scheduler_outputs, outputs, finished_before)
1388
@@ -1485,40 +1487,26 @@ class LLMEngine:
1487 scheduler_outputs = cached_outputs.scheduler_outputs
1488 allow_async_output_proc = cached_outputs.allow_async_output_proc
1489
1488 - # Detect async + multi-step
1489 - use_async_and_multi_step = (self.scheduler_config.is_multi_step
1490 - and allow_async_output_proc)
1491 -
1490 ctx = self.scheduler_contexts[virtual_engine]
1491
1492 + # Clear outputs for each new scheduler iteration
1493 + ctx.request_outputs.clear()
1494 +
1495 # Skip the scheduler if there are any remaining steps in the seq groups.
1496 # This ensures that the scheduler is only called again when the current
1497 # batch has completed.
1498 if not self._has_remaining_steps(seq_group_metadata_list):
1498 -
1499 - # Clear outputs on scheduler iteration start
1500 - ctx.request_outputs.clear()
1501 -
1499 # Schedule iteration
1500 (seq_group_metadata_list, scheduler_outputs,
1501 allow_async_output_proc
1502 ) = self.scheduler[virtual_engine].schedule()
1503
1507 - # Detect async + multi-step
1508 - use_async_and_multi_step = (self.scheduler_config.is_multi_step
1509 - and allow_async_output_proc)
1504 + ctx.seq_group_metadata_list = seq_group_metadata_list
1505 + ctx.scheduler_outputs = scheduler_outputs
1506
1507 # Maybe switch from async mode to sync mode
1508 if not allow_async_output_proc and len(ctx.output_queue) > 0:
1513 - self._process_model_outputs(virtual_engine=virtual_engine,
1514 - is_async=True)
1515 -
1516 - # For async + multi-step, init the queue
1517 - if use_async_and_multi_step:
1518 - assert len(ctx.output_queue) == 0
1519 - assert seq_group_metadata_list is not None
1520 - ctx.output_queue.append(
1521 - (None, seq_group_metadata_list, scheduler_outputs))
1509 + self._process_model_outputs(ctx=ctx)
1510
1511 if (self.scheduler_config.is_multi_step
1512 and scheduler_outputs.num_lookahead_slots > 0):
@@ -1555,13 +1543,8 @@ class LLMEngine:
1543 last_sampled_token_ids=last_sampled_token_ids)
1544
1545 if allow_async_output_proc:
1558 - async_callback = self.async_callback_multi_step[
1559 - virtual_engine] if use_async_and_multi_step \
1560 - else self.async_callback[virtual_engine]
1561 -
1562 - execute_model_req.async_callback = async_callback
1563 - execute_model_req.use_async_and_multi_step = \
1564 - use_async_and_multi_step
1546 + execute_model_req.async_callback = self.async_callbacks[
1547 + virtual_engine]
1548
1549 output = self.model_executor.execute_model(
1550 execute_model_req=execute_model_req)
@@ -1573,10 +1556,8 @@ class LLMEngine:
1556 else:
1557 # Nothing scheduled => If there is pending async postprocessor,
1558 # then finish it here.
1576 - if not use_async_and_multi_step and len(ctx.output_queue) > 0:
1577 - assert not self.scheduler_config.is_multi_step
1578 - self._process_model_outputs(virtual_engine=virtual_engine,
1579 - is_async=True)
1559 + if len(ctx.output_queue) > 0:
1560 + self._process_model_outputs(ctx=ctx)
1561 # No outputs in this case
1562 output = []
1563
@@ -1590,28 +1571,24 @@ class LLMEngine:
1571 if self.scheduler_config.is_multi_step:
1572 self.cached_scheduler_outputs[0] = SchedulerOutputState()
1573
1593 - if use_async_and_multi_step:
1594 - # For async + multi-step, clear the queue
1595 - ctx.output_queue.clear()
1596 - else:
1597 - # Add results to the output_queue
1598 - # (for async or non-async postprocessing)
1599 - ctx.output_queue.append(
1600 - (output, seq_group_metadata_list, scheduler_outputs))
1574 + # Add results to the output_queue
1575 + is_async = allow_async_output_proc
1576 + is_last_step = True
1577 + ctx.output_queue.append(
1578 + (output, seq_group_metadata_list, scheduler_outputs, is_async,
1579 + is_last_step))
1580
1602 - if output and allow_async_output_proc:
1603 - assert len(output) == 1, (
1604 - "Multi step decoding does not work "
1605 - "with async output processing.")
1581 + if output and allow_async_output_proc:
1582 + assert len(output) == 1, (
1583 + "Async postprocessor expects only a single output set")
1584
1607 - self._advance_to_next_step(
1608 - output[0], seq_group_metadata_list,
1609 - scheduler_outputs.scheduled_seq_groups)
1585 + self._advance_to_next_step(
1586 + output[0], seq_group_metadata_list,
1587 + scheduler_outputs.scheduled_seq_groups)
1588
1589 # Check if need to run the usual non-async path
1590 if not allow_async_output_proc:
1613 - self._process_model_outputs(virtual_engine=virtual_engine,
1614 - is_async=False)
1591 + self._process_model_outputs(ctx=ctx)
1592
1593 # Log stats.
1594 self.do_log_stats(scheduler_outputs, output)
@@ -1620,17 +1597,12 @@ class LLMEngine:
1597 self.do_tracing(scheduler_outputs)
1598 else:
1599 # Multi-step case
1623 - if use_async_and_multi_step:
1624 - return []
1625 - else:
1626 - ctx.request_outputs = []
1600 + return ctx.request_outputs
1601
1602 if not self.has_unfinished_requests():
1603 # Drain async postprocessor (if exists)
1604 if len(ctx.output_queue) > 0:
1631 - assert not self.scheduler_config.is_multi_step
1632 - self._process_model_outputs(virtual_engine=virtual_engine,
1633 - is_async=True)
1605 + self._process_model_outputs(ctx=ctx)
1606 assert len(ctx.output_queue) == 0
1607
1608 # Stop the execute model loop in parallel workers until there are
vllm/engine/output_processor/multi_step.py
+36 -26
@@ -85,9 +85,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
85 no tokens need to be appended since it is already done
86 externally (before the next schedule() call)
87 """
88 - # TODO: Add support for async if necessary
89 - assert not is_async
90 -
88 # Sequences can be in RUNNING or FINISHED_ABORTED state
89 # once scheduled, as a sequence is moved to FINSIHED_ABORTED
90 # if a client disconnects from the api server.
@@ -101,19 +98,41 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
98 "Beam search not supported in multi-step decoding.")
99 seq = seqs[0]
100
104 - # Since there's only one sequence per sequence group, we can take the
105 - # first sample.
106 - samples = [output.samples[0] for output in outputs]
107 -
108 - # -1 means the output token is not valid (eg. due to spec decode
109 - # rejecting tokens).
110 - valid_samples = [
111 - sample for sample in samples if sample.output_token != -1
112 - ]
113 - assert valid_samples
114 -
115 - self._process_seq_outputs(seq, valid_samples,
116 - sequence_group.sampling_params)
101 + if is_async:
102 + # Async case: We process tokens one by one. Here, we know the token
103 + # was already appended, so we only need to do the rest of the
104 + # postprocessor: Detokenization + stopping logic
105 + self._process_decode_and_stop(seq, sequence_group.sampling_params)
106 + else:
107 + # Standard multi-step case
108 +
109 + # Since there's only one sequence per sequence group,
110 + # we can take the first sample.
111 + samples = [output.samples[0] for output in outputs]
112 +
113 + # -1 means the output token is not valid (eg. due to spec decode
114 + # rejecting tokens).
115 + valid_samples = [
116 + sample for sample in samples if sample.output_token != -1
117 + ]
118 + assert valid_samples
119 +
120 + self._process_seq_outputs(seq, valid_samples,
121 + sequence_group.sampling_params)
122 +
123 + def _process_decode_and_stop(self, seq: Sequence,
124 + sampling_params: SamplingParams) -> None:
125 + new_char_count = 0
126 + if sampling_params.detokenize:
127 + new_char_count = self.detokenizer.decode_sequence_inplace(
128 + seq, sampling_params)
129 +
130 + # TODO(sang): Support lora.
131 + self.stop_checker.maybe_stop_sequence(
132 + seq,
133 + new_char_count=new_char_count,
134 + sampling_params=sampling_params,
135 + )
136
137 def _process_seq_outputs(self, seq: Sequence,
138 valid_samples: List[SequenceOutput],
@@ -151,16 +170,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
170 logprobs=output_logprob,
171 )
172
154 - new_char_count = 0
155 - if sampling_params.detokenize:
156 - new_char_count = self.detokenizer.decode_sequence_inplace(
157 - seq, sampling_params)
173 + self._process_decode_and_stop(seq, sampling_params)
174
159 - # TODO(sang): Support lora.
160 - self.stop_checker.maybe_stop_sequence(
161 - seq,
162 - new_char_count=new_char_count,
163 - sampling_params=sampling_params,
164 - )
175 if seq.is_finished():
176 break
vllm/sequence.py
+1 -3
@@ -1225,7 +1225,6 @@ class ExecuteModelRequest(
1225 last_sampled_token_ids: Optional[torch.Tensor] = None
1226 # Async callback
1227 async_callback: Optional[Callable] = None
1228 - use_async_and_multi_step: bool = False
1228
1229 @property
1230 def is_first_multi_step(self) -> bool:
@@ -1272,5 +1271,4 @@ class ExecuteModelRequest(
1271 finished_requests_ids=self.finished_requests_ids,
1272 last_sampled_token_ids=self.last_sampled_token_ids.clone()
1273 if self.last_sampled_token_ids is not None else None,
1275 - async_callback=self.async_callback,
1276 - use_async_and_multi_step=self.use_async_and_multi_step)
1274 + async_callback=self.async_callback)
vllm/worker/model_runner.py
+3 -1
@@ -21,6 +21,7 @@ from vllm.attention.backends.utils import CommonAttentionState
21 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
22 ModelConfig, ObservabilityConfig, ParallelConfig,
23 PromptAdapterConfig, SchedulerConfig)
24 +from vllm.core.scheduler import SchedulerOutputs
25 from vllm.distributed import get_pp_group
26 from vllm.distributed.parallel_state import graph_capture
27 from vllm.inputs import INPUT_REGISTRY, InputRegistry
@@ -96,7 +97,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
97 finished_requests_ids: Optional[List[str]] = None
98 virtual_engine: int = 0
99 async_callback: Optional[Callable] = None
99 - use_async_and_multi_step: bool = False
100 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None
101 + scheduler_outputs: Optional[SchedulerOutputs] = None
102
103 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
104 tensor_dict = {
vllm/worker/multi_step_model_runner.py
+132 -33
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
22 get_pythonized_sample_results)
23 from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
24 Logprob, SequenceGroupMetadata, SequenceOutput)
25 +from vllm.utils import PyObjectCache
26 from vllm.worker.model_runner import (GPUModelRunnerBase,
27 ModelInputForGPUWithSamplingMetadata)
28 from vllm.worker.model_runner_base import (
@@ -37,6 +38,29 @@ if TYPE_CHECKING:
38 logger = init_logger(__name__)
39
40
41 +def seq_output_builder():
42 + return SequenceOutput(
43 + 0, 0,
44 + {0: Logprob(logprob=float('inf'), rank=None, decoded_token=None)})
45 +
46 +
47 +def completion_seq_group_output_builder():
48 + return CompletionSequenceGroupOutput([], None)
49 +
50 +
51 +# Used by pythonization to reduce python object allocations
52 +class PythonizationCache:
53 +
54 + def __init__(self):
55 + self.cached_seq_output = PyObjectCache(seq_output_builder)
56 + self.cached_completion_seq_group_output = PyObjectCache(
57 + completion_seq_group_output_builder)
58 +
59 + def reset(self):
60 + self.cached_seq_output.reset()
61 + self.cached_completion_seq_group_output.reset()
62 +
63 +
64 @dataclass
65 class ModelOutput:
66 """The output of a single model forward pass.
@@ -59,6 +83,7 @@ class ModelOutput:
83 pythonized: bool = False
84 # On-device tensor containing the logprobs of each token.
85 logprobs: Optional["torch.Tensor"] = None
86 + pythonization_cache: Optional[PythonizationCache] = None
87
88 def pythonize(self, input_metadata: "StatefulModelInput",
89 copy_stream: torch.cuda.Stream,
@@ -97,7 +122,8 @@ class ModelOutput:
122 with torch.cuda.stream(copy_stream):
123 _pythonize_sampler_output(input_metadata, self.sampler_output,
124 pinned_sampled_token_buffer,
100 - self.sampled_token_ids, self.logprobs)
125 + self.sampled_token_ids, self.logprobs,
126 + self.pythonization_cache)
127
128 # Erase the logprobs GPU-side tensor.
129 # Note that although _pythonize_sampler_output() runs in its
@@ -209,6 +235,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
235 self._copy_stream = torch.cuda.Stream()
236 self.pinned_sampled_token_ids: Optional[torch.Tensor] = None
237
238 + self.pythonization_cache = PythonizationCache()
239 +
240 def make_model_input_from_broadcasted_tensor_dict(
241 self, tensor_dict: Dict[str, Any]) -> StatefulModelInput:
242 model_input = (StatefulModelInput.from_broadcasted_tensor_dict(
@@ -237,14 +265,22 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
265 output_proc_callback: Callable):
266 # Proceed with pythonization and output_proc in order.
267 # Stop on the first one that fails to pythonize
268 + output_proc_callback()
269 +
270 cont = True
271 for model_output in model_input.cached_outputs:
272 if not model_output.pythonized:
273 model_output.maybe_pythonize(model_input, self._copy_stream,
274 self.pinned_sampled_token_ids)
275 if model_output.pythonized:
246 - output_proc_callback(
247 - sampler_output=model_output.sampler_output)
276 + ctx = output_proc_callback.keywords["ctx"]
277 + is_async = False
278 + is_last_step = False
279 + ctx.output_queue.append(
280 + ([model_output.sampler_output
281 + ], ctx.seq_group_metadata_list,
282 + ctx.scheduler_outputs, is_async, is_last_step))
283 + output_proc_callback()
284 else:
285 cont = False
286
@@ -255,21 +291,46 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
291 output_proc_callback: Optional[Callable]):
292 assert model_input.frozen_model_input is not None
293
294 + has_async_callback = output_proc_callback is not None
295 +
296 outputs = []
297 for output_id in range(len(model_input.cached_outputs)):
260 - is_last_output = output_id == len(model_input.cached_outputs) - 1
261 -
298 output = model_input.cached_outputs[output_id]
263 - if not output.pythonized:
299 + is_last_step = output_id == len(model_input.cached_outputs) - 1
300 +
301 + # For non-async case:
302 + # -- We simply add the outputs
303 + # For async case:
304 + # -- Invoke callback, pythonize, add to callback queue and repeat
305 + # -- For last output, just add to callback queue
306 + if has_async_callback:
307 + assert output_proc_callback is not None
308 +
309 + # Invoke callback before pythonize (to overlap with GPU)
310 + output_proc_callback()
311 +
312 + # Pythonize
313 + if not output.pythonized:
314 + output.pythonize(model_input, self._copy_stream,
315 + self.pinned_sampled_token_ids)
316 +
317 + # For non last step, add to callback queue to chain
318 + # callbacks=>pythonize pairs (for GPU overlap)
319 + if not is_last_step:
320 + ctx = output_proc_callback.keywords[ # type: ignore
321 + "ctx"] # type: ignore
322 + is_async = False
323 + is_last_step = False
324 + ctx.output_queue.append(
325 + ([output.sampler_output
326 + ], ctx.seq_group_metadata_list,
327 + ctx.scheduler_outputs, is_async, is_last_step))
328 + else:
329 + outputs.append(output.sampler_output)
330 + else:
331 output.pythonize(model_input, self._copy_stream,
332 self.pinned_sampled_token_ids)
266 -
267 - if model_input.frozen_model_input.use_async_and_multi_step:
268 - assert output_proc_callback is not None
269 - output_proc_callback(sampler_output=output.sampler_output,
270 - is_last_output=is_last_output)
271 -
272 - outputs.append(output.sampler_output)
333 + outputs.append(output.sampler_output)
334
335 return outputs
336
@@ -330,7 +391,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
391 model_input, model_input.cached_outputs[-1].sampler_output)
392
393 output_proc_callback = None
333 - if frozen_model_input.use_async_and_multi_step:
394 + if frozen_model_input.async_callback is not None:
395 output_proc_callback = frozen_model_input.async_callback
396 assert output_proc_callback is not None
397 async_callback = functools.partial(
@@ -367,7 +428,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
428 model_input.cached_outputs.append(
429 ModelOutput(output[0], output_ready_event,
430 output[0].sampled_token_ids, False,
370 - output[0].logprobs))
431 + output[0].logprobs, self.pythonization_cache))
432
433 # These GPU tensors are not required by multi-step;
434 # erase them to ensure they are not pythonized or
@@ -378,7 +439,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
439
440 # Pythonize the output if CPU is ahead and the previous step is
441 # ready.
381 - if not frozen_model_input.use_async_and_multi_step:
442 + if frozen_model_input.async_callback is None:
443 for model_output in model_input.cached_outputs:
444 model_output.maybe_pythonize(model_input,
445 self._copy_stream,
@@ -397,6 +458,7 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
458 if model_input.is_last_step:
459 outputs = self._final_process_outputs(model_input,
460 output_proc_callback)
461 + self.pythonization_cache.reset()
462 return outputs
463
464 # should be [SamplerOutput]
@@ -537,6 +599,7 @@ def _pythonize_sampler_output(
599 pinned_sampled_token_buffer: torch.Tensor,
600 sampled_token_ids: torch.Tensor,
601 logprobs_tensor: Optional[torch.Tensor],
602 + cache: Optional[PythonizationCache],
603 ) -> None:
604 """ This function is only called when the output tensors are ready.
605 See :class:`ModelOutput`.
@@ -597,6 +660,9 @@ def _pythonize_sampler_output(
660
661 for sgdx, (seq_group,
662 sample_result) in enumerate(zip(seq_groups, samples_list)):
663 + if seq_group.sampling_params.logits_processors:
664 + assert len(seq_group.sampling_params.logits_processors) == 0, (
665 + "Logits Processors are not supported in multi-step decoding")
666
667 if do_pythonize_logprobs:
668 assert prompt_logprobs is not None
@@ -621,23 +687,56 @@ def _pythonize_sampler_output(
687 seq_ids = seq_group.seq_ids
688 next_token_ids = sample_result
689 parent_ids = [0]
624 - seq_outputs: List[SequenceOutput] = []
625 - if seq_group.sampling_params.logits_processors:
626 - assert len(seq_group.sampling_params.logits_processors) == 0, (
627 - "Logits Processors are not supported in multi-step decoding")
690 +
691 + if cache is not None:
692 + completion_seq_group_output: CompletionSequenceGroupOutput = \
693 + cache.cached_completion_seq_group_output.get_object()
694 + completion_seq_group_output.samples.clear()
695 + seq_outputs: List[
696 + SequenceOutput] = completion_seq_group_output.samples
697 + else:
698 + seq_outputs = []
699 +
700 for tdx, (parent_id,
701 next_token_id) in enumerate(zip(parent_ids, next_token_ids)):
630 - seq_outputs.append(
631 - SequenceOutput(seq_ids[parent_id], next_token_id,
632 - (group_sample_logprobs[tdx]
633 - if logprobs_are_requested else {
634 - next_token_id:
635 - Logprob(logprob=float('inf'),
636 - rank=None,
637 - decoded_token=None)
638 - })))
639 - output.outputs.append(
640 - CompletionSequenceGroupOutput(
641 - seq_outputs,
642 - (group_prompt_logprobs if logprobs_are_requested else None)))
702 + if cache is not None:
703 + seq_output: SequenceOutput = cache.cached_seq_output.get_object(
704 + )
705 + seq_output.parent_seq_id = seq_ids[parent_id]
706 + seq_output.output_token = next_token_id
707 +
708 + if logprobs_are_requested:
709 + seq_output.logprobs = group_sample_logprobs[tdx]
710 + else:
711 + logprobs = next(iter(seq_output.logprobs.values()))
712 + seq_output.logprobs.clear()
713 +
714 + logprobs.logprob = float('inf')
715 + logprobs.rank = None
716 + logprobs.decoded_token = None
717 +
718 + seq_output.logprobs[next_token_id] = logprobs
719 +
720 + seq_outputs.append(seq_output)
721 +
722 + else:
723 + seq_outputs.append(
724 + SequenceOutput(seq_ids[parent_id], next_token_id,
725 + (group_sample_logprobs[tdx]
726 + if logprobs_are_requested else {
727 + next_token_id:
728 + Logprob(logprob=float('inf'),
729 + rank=None,
730 + decoded_token=None)
731 + })))
732 + if cache is not None:
733 + completion_seq_group_output.prompt_logprobs = \
734 + group_prompt_logprobs if logprobs_are_requested else None
735 + output.outputs.append(completion_seq_group_output)
736 + else:
737 + output.outputs.append(
738 + CompletionSequenceGroupOutput(
739 + seq_outputs, (group_prompt_logprobs
740 + if logprobs_are_requested else None)))
741 +
742 assert len(output.outputs) > 0
vllm/worker/multi_step_worker.py
+1 -3
@@ -67,9 +67,7 @@ class MultiStepWorker(Worker):
67 if execute_model_req.async_callback:
68 model_input.frozen_model_input = dataclasses.replace( # type: ignore
69 model_input.frozen_model_input,
70 - async_callback=execute_model_req.async_callback,
71 - use_async_and_multi_step=execute_model_req.
72 - use_async_and_multi_step)
70 + async_callback=execute_model_req.async_callback)
71 else:
72 # on subsequent steps we reuse the worker input and model input
73 multi_step_state = self.multi_step_states[virtual_engine]