A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
align llm_engine and async_engine. (#1081)
Roy committed
Sep 19, 2023 at 02:49 UTC
95592fa00a01c1e779137f72814c8091736e1c86
1 file changed
+4
-5
vllm/engine/async_llm_engine.py
+4
-5
@@ -183,10 +183,9 @@ class _AsyncLLMEngine(LLMEngine):
183
and updates the scheduler with the model outputs. Finally, it decodes
184
the sequences and returns the newly generated results.
185
"""
186
- (seq_group_metadata_list, scheduler_outputs,
187
- early_return) = self._schedule()
188
- if early_return is not None:
189
- return early_return
186
+ seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
187
+ if scheduler_outputs.is_empty():
188
+ return ignored
189
190
# Execute the model.
191
output = await self._run_workers_async(
@@ -197,7 +196,7 @@ class _AsyncLLMEngine(LLMEngine):
196
blocks_to_copy=scheduler_outputs.blocks_to_copy,
197
)
198
200
- return self._process_model_outputs(output, scheduler_outputs)
199
+ return self._process_model_outputs(output, scheduler_outputs) + ignored
200
201
async def _run_workers_async(
202
self,