A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[BUGFIX] Do not return ignored sentences twice in async llm engine (#2258)
Zhuohan Li committed
Dec 26, 2023 at 13:41 UTC
e0ff920001988f2240cf05551bef566898ed6e4b
2 files changed
+7
-22
vllm/engine/async_llm_engine.py
+4
-6
@@ -183,20 +183,18 @@ class _AsyncLLMEngine(LLMEngine):
183
and updates the scheduler with the model outputs. Finally, it decodes
184
the sequences and returns the newly generated results.
185
"""
186
- seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
187
- if scheduler_outputs.is_empty():
188
- return ignored
186
+ seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
187
188
# Execute the model.
191
- output = await self._run_workers_async(
189
+ output = (await self._run_workers_async(
190
"execute_model",
191
seq_group_metadata_list=seq_group_metadata_list,
192
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
193
blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
194
blocks_to_copy=scheduler_outputs.blocks_to_copy,
197
- )
195
+ )) if not scheduler_outputs.is_empty() else []
196
199
- return self._process_model_outputs(output, scheduler_outputs) + ignored
197
+ return self._process_model_outputs(output, scheduler_outputs)
198
199
async def _run_workers_async(
200
self,
vllm/engine/llm_engine.py
+3
-16
@@ -14,8 +14,7 @@ from vllm.logger import init_logger
14
from vllm.outputs import RequestOutput
15
from vllm.sampling_params import SamplingParams
16
from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
17
- SequenceGroupMetadata, SequenceGroupOutput,
18
- SequenceOutput, SequenceStatus)
17
+ SequenceGroupOutput, SequenceOutput, SequenceStatus)
18
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
19
get_tokenizer)
20
from vllm.utils import Counter
@@ -328,16 +327,6 @@ class LLMEngine:
327
"""Returns True if there are unfinished requests."""
328
return self.scheduler.has_unfinished_seqs()
329
331
- def _schedule(
332
- self
333
- ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
334
- List[RequestOutput]]:
335
- seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
336
- return seq_group_metadata_list, scheduler_outputs, [
337
- RequestOutput.from_seq_group(seq_group)
338
- for seq_group in scheduler_outputs.ignored_seq_groups
339
- ]
340
-
330
def _check_beam_search_early_stopping(
331
self,
332
early_stopping: Union[bool, str],
@@ -586,9 +575,7 @@ class LLMEngine:
575
and updates the scheduler with the model outputs. Finally, it decodes
576
the sequences and returns the newly generated results.
577
"""
589
- seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
590
- if scheduler_outputs.is_empty():
591
- return ignored
578
+ seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
579
580
# Execute the model.
581
output = self._run_workers(
@@ -597,7 +584,7 @@ class LLMEngine:
584
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
585
blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
586
blocks_to_copy=scheduler_outputs.blocks_to_copy,
600
- )
587
+ ) if not scheduler_outputs.is_empty() else []
588
589
return self._process_model_outputs(output, scheduler_outputs)
590