A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[BugFix] get_and_reset only when scheduler outputs are not empty (#6266)
Mor Zusman committed
Jul 11, 2024 at 17:40 UTC
55f692b46ef35ed4a9e199dfe60a9eefe800e4b0
2 files changed
+4
-4
vllm/engine/async_llm_engine.py
+2
-2
@@ -225,11 +225,11 @@ class _AsyncLLMEngine(LLMEngine):
225
"""
226
seq_group_metadata_list, scheduler_outputs = self.scheduler[
227
virtual_engine].schedule()
228
- finished_requests_ids = self.scheduler[
229
- virtual_engine].get_and_reset_finished_requests_ids()
228
229
if not scheduler_outputs.is_empty():
230
# Execute the model.
231
+ finished_requests_ids = self.scheduler[
232
+ virtual_engine].get_and_reset_finished_requests_ids()
233
execute_model_req = ExecuteModelRequest(
234
seq_group_metadata_list=seq_group_metadata_list,
235
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
vllm/engine/llm_engine.py
+2
-2
@@ -871,10 +871,10 @@ class LLMEngine:
871
"as performance will be severely degraded otherwise.")
872
seq_group_metadata_list, scheduler_outputs = self.scheduler[
873
0].schedule()
874
- finished_requests_ids = self.scheduler[
875
- 0].get_and_reset_finished_requests_ids()
874
875
if not scheduler_outputs.is_empty():
876
+ finished_requests_ids = self.scheduler[
877
+ 0].get_and_reset_finished_requests_ids()
878
execute_model_req = ExecuteModelRequest(
879
seq_group_metadata_list=seq_group_metadata_list,
880
blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,