A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[V1][Scheduler] Avoid calling `_try_schedule_encoder_inputs` for every request (#15778)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Mar 30, 2025 at 14:10 UTC
9b459eca88b4953586391c14d183574c4d21fca3
2 files changed
+30
-30
vllm/v1/core/sched/scheduler.py
+28
-23
@@ -171,19 +171,23 @@ class Scheduler(SchedulerInterface):
171
assert num_new_tokens > 0
172
173
# Schedule encoder inputs.
174
- encoder_inputs_to_schedule, num_new_tokens, new_encoder_budget = (
175
- self._try_schedule_encoder_inputs(request,
176
- request.num_computed_tokens,
177
- num_new_tokens,
178
- encoder_budget))
179
- if num_new_tokens == 0:
180
- # The request cannot be scheduled because the encoder budget
181
- # or the encoder cache is exhausted.
182
- # NOTE(woosuk): Here, by doing `continue` instead of `break`,
183
- # we do not strictly follow the FCFS scheduling policy and
184
- # allow the lower-priority requests to be scheduled.
185
- req_index += 1
186
- continue
174
+ if request.has_encoder_inputs:
175
+ (encoder_inputs_to_schedule, num_new_tokens,
176
+ new_encoder_budget) = self._try_schedule_encoder_inputs(
177
+ request, request.num_computed_tokens, num_new_tokens,
178
+ encoder_budget)
179
+ if num_new_tokens == 0:
180
+ # The request cannot be scheduled because the encoder budget
181
+ # or the encoder cache is exhausted.
182
+ # NOTE(woosuk): By using `continue` instead of `break` here,
183
+ # we intentionally relax the strict FCFS scheduling policy
184
+ # to allow lower-priority requests to be scheduled when a
185
+ # higher-priority request is blocked by encoder constraints.
186
+ req_index += 1
187
+ continue
188
+ else:
189
+ encoder_inputs_to_schedule = None
190
+ new_encoder_budget = encoder_budget
191
192
while True:
193
new_blocks = self.kv_cache_manager.allocate_slots(
@@ -318,13 +322,17 @@ class Scheduler(SchedulerInterface):
322
assert num_new_tokens > 0
323
324
# Schedule encoder inputs.
321
- (encoder_inputs_to_schedule, num_new_tokens,
322
- new_encoder_budget) = self._try_schedule_encoder_inputs(
323
- request, num_computed_tokens, num_new_tokens,
324
- encoder_budget)
325
- if num_new_tokens == 0:
326
- # The request cannot be scheduled.
327
- break
325
+ if request.has_encoder_inputs:
326
+ (encoder_inputs_to_schedule, num_new_tokens,
327
+ new_encoder_budget) = self._try_schedule_encoder_inputs(
328
+ request, num_computed_tokens, num_new_tokens,
329
+ encoder_budget)
330
+ if num_new_tokens == 0:
331
+ # The request cannot be scheduled.
332
+ break
333
+ else:
334
+ encoder_inputs_to_schedule = None
335
+ new_encoder_budget = encoder_budget
336
337
new_blocks = self.kv_cache_manager.allocate_slots(
338
request, num_new_tokens, computed_blocks)
@@ -506,9 +514,6 @@ class Scheduler(SchedulerInterface):
514
limitations, the method adjusts `num_new_tokens` to schedule only the
515
decoder tokens up to just before the unschedulable encoder input.
516
"""
509
- if not request.has_encoder_inputs():
510
- return [], num_new_tokens, encoder_budget
511
-
517
encoder_inputs_to_schedule: list[int] = []
518
mm_positions = request.mm_positions
519
assert mm_positions is not None
vllm/v1/request.py
+2
-7
@@ -59,6 +59,8 @@ class Request:
59
self.mm_positions = multi_modal_placeholders or []
60
self.mm_inputs = multi_modal_inputs or []
61
self.mm_hashes: list[str] = multi_modal_hashes or []
62
+ self.num_encoder_inputs = len(self.mm_inputs)
63
+ self.has_encoder_inputs = self.num_encoder_inputs > 0
64
65
# Sanity check
66
assert len(self.mm_inputs) == len(self.mm_positions)
@@ -117,13 +119,6 @@ class Request:
119
def get_finished_reason(self) -> Union[FinishReason, None]:
120
return RequestStatus.get_finished_reason(self.status)
121
120
- def has_encoder_inputs(self) -> bool:
121
- return len(self.mm_inputs) > 0
122
-
123
- @property
124
- def num_encoder_inputs(self) -> int:
125
- return len(self.mm_positions)
126
-
122
def get_num_encoder_tokens(self, input_id: int) -> int:
123
assert input_id < len(self.mm_positions)
124
num_tokens = self.mm_positions[input_id]["length"]