A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[V1][PP] Do not block engine core when no requests to schedule (#14585)
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
Cody Yu committed
Mar 10, 2025 at 19:48 UTC
4290b704ffe025de8efdd2ecce062ec8bc34e1d6
1 file changed
+11
-16
vllm/v1/engine/core.py
+11
-16
@@ -205,23 +205,18 @@ class EngineCore:
205
self.batch_queue.put_nowait(
206
(future, scheduler_output)) # type: ignore
207
208
- # If all requests are scheduled or the job queue is full,
208
+ scheduled_batch = (scheduler_output is not None
209
+ and scheduler_output.total_num_scheduled_tokens > 0)
210
+
211
+ # If no more requests can be scheduled and the job queue is not empty,
212
# block until the first batch in the job queue is finished.
210
- if (scheduler_output is None
211
- or scheduler_output.total_num_scheduled_tokens == 0):
212
- try:
213
- future, scheduler_output = self.batch_queue.get(
214
- timeout=POLLING_TIMEOUT_S)
215
- # Blocking until the first result is available.
216
- model_output = future.result()
217
- self.batch_queue.task_done()
218
- engine_core_outputs = self.scheduler.update_from_output(
219
- scheduler_output, model_output)
220
- except queue.Empty:
221
- # If the queue is empty (timeout at .get), return
222
- # an empty EngineCoreOutputs for logging.
223
- engine_core_outputs = EngineCoreOutputs(
224
- outputs=[], scheduler_stats=self.scheduler.make_stats())
213
+ if not scheduled_batch and not self.batch_queue.empty():
214
+ future, scheduler_output = self.batch_queue.get_nowait()
215
+ # Blocking until the first result is available.
216
+ model_output = future.result()
217
+ self.batch_queue.task_done()
218
+ engine_core_outputs = self.scheduler.update_from_output(
219
+ scheduler_output, model_output)
220
221
return engine_core_outputs
222