@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b7ee940a82
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1][BugFix] Fix edge case in VLM scheduling (#12065)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jan 14, 2025 at 20:21 UTC b7ee940a828de9d339345e28eee8b13d60d97f26
1 file changed +15 -11
vllm/v1/core/scheduler.py
+15 -11
@@ -373,18 +373,22 @@ class Scheduler:
373 if self.encoder_cache_manager.has_cache(request, i):
374 # The encoder input is already computed and cached.
375 continue
376 - if not self.encoder_cache_manager.can_allocate(request, i):
377 - # The encoder cache is full. We can only schedule the decoder
378 - # tokens just before the encoder input.
379 - num_new_tokens = start_pos - num_computed_tokens
380 - break
381 - if num_encoder_tokens > encoder_budget:
382 - # The encoder budget is exhausted. We can only schedule the
383 - # decoder tokens up until the encoder input.
384 - # NOTE(woosuk): We assume that the encoder tokens should be
385 - # processed altogether, as the encoder usually uses
376 + if (not self.encoder_cache_manager.can_allocate(request, i)
377 + or num_encoder_tokens > encoder_budget):
378 + # The encoder cache is full or the encoder budget is exhausted.
379 + # NOTE(woosuk): We assume that the encoder input tokens should
380 + # be processed altogether, as the encoder usually uses
381 # bidirectional attention.
387 - num_new_tokens = start_pos - num_computed_tokens
382 + if num_computed_tokens < start_pos:
383 + # We only schedule the decoder tokens just before the
384 + # encoder input.
385 + num_new_tokens = start_pos - num_computed_tokens
386 + else:
387 + # Because of prefix caching, num_computed_tokens is greater
388 + # than start_pos even though its encoder input is not
389 + # available. In this case, we can't schedule any token for
390 + # the request in this step.
391 + num_new_tokens = 0
392 break
393
394 encoder_budget -= num_encoder_tokens