A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 10:00 UTC
[Bugfix] Whisper fix number of allocated CrossAttn blocks per-request (#30772)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Dec 16, 2025 at 15:20 UTC
75eb302a2e4000470d1ad6bfc3a009379554b648
1 file changed
+11
-11
vllm/v1/core/sched/scheduler.py
+11
-11
@@ -187,6 +187,12 @@ class Scheduler(SchedulerInterface):
187
if self.is_encoder_decoder
188
else EncoderCacheManager(cache_size=encoder_cache_size)
189
)
190
+ # For encoder-decoder models, allocate the maximum number of tokens for Cross
191
+ # Attn blocks, as for Whisper its input is always padded to the maximum length.
192
+ # TODO (NickLucche): Generalize to models with variable-length encoder inputs.
193
+ self._num_encoder_max_input_tokens = (
194
+ MULTIMODAL_REGISTRY.get_encdec_max_encoder_len(vllm_config.model_config)
195
+ )
196
197
speculative_config = vllm_config.speculative_config
198
self.use_eagle = False
@@ -568,17 +574,11 @@ class Scheduler(SchedulerInterface):
574
0 if request.num_computed_tokens == 0 else self.num_lookahead_tokens
575
)
576
571
- # Determine if we need to allocate cross-attention blocks.
572
- if self.is_encoder_decoder and request.has_encoder_inputs:
573
- # TODO(russellb): For Whisper, we know that the input is
574
- # always padded to the maximum length. If we support other
575
- # encoder-decoder models, this will need to be updated if we
576
- # want to only allocate what is needed.
577
- num_encoder_tokens = (
578
- self.scheduler_config.max_num_encoder_input_tokens
579
- )
580
- else:
581
- num_encoder_tokens = 0
577
+ num_encoder_tokens = (
578
+ self._num_encoder_max_input_tokens
579
+ if self.is_encoder_decoder and request.has_encoder_inputs
580
+ else 0
581
+ )
582
583
new_blocks = self.kv_cache_manager.allocate_slots(
584
request,