@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 95592fa00a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

align llm_engine and async_engine. (#1081)

Roy committed Sep 19, 2023 at 02:49 UTC 95592fa00a01c1e779137f72814c8091736e1c86
1 file changed +4 -5
vllm/engine/async_llm_engine.py
+4 -5
@@ -183,10 +183,9 @@ class _AsyncLLMEngine(LLMEngine):
183 and updates the scheduler with the model outputs. Finally, it decodes
184 the sequences and returns the newly generated results.
185 """
186 - (seq_group_metadata_list, scheduler_outputs,
187 - early_return) = self._schedule()
188 - if early_return is not None:
189 - return early_return
186 + seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
187 + if scheduler_outputs.is_empty():
188 + return ignored
189
190 # Execute the model.
191 output = await self._run_workers_async(
@@ -197,7 +196,7 @@ class _AsyncLLMEngine(LLMEngine):
196 blocks_to_copy=scheduler_outputs.blocks_to_copy,
197 )
198
200 - return self._process_model_outputs(output, scheduler_outputs)
199 + return self._process_model_outputs(output, scheduler_outputs) + ignored
200
201 async def _run_workers_async(
202 self,