A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1][BugFix] Fix edge case in VLM scheduling (#12065)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jan 14, 2025 at 20:21 UTC
b7ee940a828de9d339345e28eee8b13d60d97f26
1 file changed
+15
-11
vllm/v1/core/scheduler.py
+15
-11
@@ -373,18 +373,22 @@ class Scheduler:
373
if self.encoder_cache_manager.has_cache(request, i):
374
# The encoder input is already computed and cached.
375
continue
376
- if not self.encoder_cache_manager.can_allocate(request, i):
377
- # The encoder cache is full. We can only schedule the decoder
378
- # tokens just before the encoder input.
379
- num_new_tokens = start_pos - num_computed_tokens
380
- break
381
- if num_encoder_tokens > encoder_budget:
382
- # The encoder budget is exhausted. We can only schedule the
383
- # decoder tokens up until the encoder input.
384
- # NOTE(woosuk): We assume that the encoder tokens should be
385
- # processed altogether, as the encoder usually uses
376
+ if (not self.encoder_cache_manager.can_allocate(request, i)
377
+ or num_encoder_tokens > encoder_budget):
378
+ # The encoder cache is full or the encoder budget is exhausted.
379
+ # NOTE(woosuk): We assume that the encoder input tokens should
380
+ # be processed altogether, as the encoder usually uses
381
# bidirectional attention.
387
- num_new_tokens = start_pos - num_computed_tokens
382
+ if num_computed_tokens < start_pos:
383
+ # We only schedule the decoder tokens just before the
384
+ # encoder input.
385
+ num_new_tokens = start_pos - num_computed_tokens
386
+ else:
387
+ # Because of prefix caching, num_computed_tokens is greater
388
+ # than start_pos even though its encoder input is not
389
+ # available. In this case, we can't schedule any token for
390
+ # the request in this step.
391
+ num_new_tokens = 0
392
break
393
394
encoder_budget -= num_encoder_tokens