@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e0ff920001
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[BUGFIX] Do not return ignored sentences twice in async llm engine (#2258)

Zhuohan Li committed Dec 26, 2023 at 13:41 UTC e0ff920001988f2240cf05551bef566898ed6e4b
2 files changed +7 -22
vllm/engine/async_llm_engine.py
+4 -6
@@ -183,20 +183,18 @@ class _AsyncLLMEngine(LLMEngine):
183 and updates the scheduler with the model outputs. Finally, it decodes
184 the sequences and returns the newly generated results.
185 """
186 - seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
187 - if scheduler_outputs.is_empty():
188 - return ignored
186 + seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
187
188 # Execute the model.
191 - output = await self._run_workers_async(
189 + output = (await self._run_workers_async(
190 "execute_model",
191 seq_group_metadata_list=seq_group_metadata_list,
192 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
193 blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
194 blocks_to_copy=scheduler_outputs.blocks_to_copy,
197 - )
195 + )) if not scheduler_outputs.is_empty() else []
196
199 - return self._process_model_outputs(output, scheduler_outputs) + ignored
197 + return self._process_model_outputs(output, scheduler_outputs)
198
199 async def _run_workers_async(
200 self,
vllm/engine/llm_engine.py
+3 -16
@@ -14,8 +14,7 @@ from vllm.logger import init_logger
14 from vllm.outputs import RequestOutput
15 from vllm.sampling_params import SamplingParams
16 from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
17 - SequenceGroupMetadata, SequenceGroupOutput,
18 - SequenceOutput, SequenceStatus)
17 + SequenceGroupOutput, SequenceOutput, SequenceStatus)
18 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
19 get_tokenizer)
20 from vllm.utils import Counter
@@ -328,16 +327,6 @@ class LLMEngine:
327 """Returns True if there are unfinished requests."""
328 return self.scheduler.has_unfinished_seqs()
329
331 - def _schedule(
332 - self
333 - ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
334 - List[RequestOutput]]:
335 - seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
336 - return seq_group_metadata_list, scheduler_outputs, [
337 - RequestOutput.from_seq_group(seq_group)
338 - for seq_group in scheduler_outputs.ignored_seq_groups
339 - ]
340 -
330 def _check_beam_search_early_stopping(
331 self,
332 early_stopping: Union[bool, str],
@@ -586,9 +575,7 @@ class LLMEngine:
575 and updates the scheduler with the model outputs. Finally, it decodes
576 the sequences and returns the newly generated results.
577 """
589 - seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
590 - if scheduler_outputs.is_empty():
591 - return ignored
578 + seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
579
580 # Execute the model.
581 output = self._run_workers(
@@ -597,7 +584,7 @@ class LLMEngine:
584 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
585 blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
586 blocks_to_copy=scheduler_outputs.blocks_to_copy,
600 - )
587 + ) if not scheduler_outputs.is_empty() else []
588
589 return self._process_model_outputs(output, scheduler_outputs)
590