@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55f692b46e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[BugFix] get_and_reset only when scheduler outputs are not empty (#6266)

Mor Zusman committed Jul 11, 2024 at 17:40 UTC 55f692b46ef35ed4a9e199dfe60a9eefe800e4b0
2 files changed +4 -4
vllm/engine/async_llm_engine.py
+2 -2
@@ -225,11 +225,11 @@ class _AsyncLLMEngine(LLMEngine):
225 """
226 seq_group_metadata_list, scheduler_outputs = self.scheduler[
227 virtual_engine].schedule()
228 - finished_requests_ids = self.scheduler[
229 - virtual_engine].get_and_reset_finished_requests_ids()
228
229 if not scheduler_outputs.is_empty():
230 # Execute the model.
231 + finished_requests_ids = self.scheduler[
232 + virtual_engine].get_and_reset_finished_requests_ids()
233 execute_model_req = ExecuteModelRequest(
234 seq_group_metadata_list=seq_group_metadata_list,
235 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
vllm/engine/llm_engine.py
+2 -2
@@ -871,10 +871,10 @@ class LLMEngine:
871 "as performance will be severely degraded otherwise.")
872 seq_group_metadata_list, scheduler_outputs = self.scheduler[
873 0].schedule()
874 - finished_requests_ids = self.scheduler[
875 - 0].get_and_reset_finished_requests_ids()
874
875 if not scheduler_outputs.is_empty():
876 + finished_requests_ids = self.scheduler[
877 + 0].get_and_reset_finished_requests_ids()
878 execute_model_req = ExecuteModelRequest(
879 seq_group_metadata_list=seq_group_metadata_list,
880 blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,