A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core] Get num_encoder_tokens from scheduler config (#24989)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Sep 16, 2025 at 23:59 UTC
58d4c705a88adc3187591b4b2e651eae3b190061
3 files changed
+6
-8
vllm/v1/core/sched/scheduler.py
+2
-3
@@ -465,9 +465,8 @@ class Scheduler(SchedulerInterface):
465
in self.vllm_config.model_config.model.lower()), (
466
"Whisper is the only supported "
467
"encoder-decoder model.")
468
- num_encoder_tokens = MULTIMODAL_REGISTRY.\
469
- get_encdec_max_encoder_len(
470
- self.vllm_config.model_config)
468
+ num_encoder_tokens =\
469
+ self.scheduler_config.max_num_encoder_input_tokens
470
else:
471
num_encoder_tokens = 0
472
vllm/v1/kv_cache_interface.py
+2
-3
@@ -11,7 +11,6 @@ from typing_extensions import Self
11
12
from vllm.config import VllmConfig
13
from vllm.logger import init_logger
14
-from vllm.multimodal import MULTIMODAL_REGISTRY
14
from vllm.utils import cdiv, get_dtype_size
15
16
logger = init_logger(__name__)
@@ -230,8 +229,8 @@ class CrossAttentionSpec(AttentionSpec):
229
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
230
# For cross-attention, we need to cache encoder states
231
# Get encoder length (e.g., 1500 for Whisper).
233
- max_encoder_len = MULTIMODAL_REGISTRY.\
234
- get_encdec_max_encoder_len(vllm_config.model_config)
232
+ max_encoder_len = vllm_config.scheduler_config.\
233
+ max_num_encoder_input_tokens
234
return cdiv(max_encoder_len, self.block_size) * self.page_size_bytes
235
236
vllm/v1/worker/gpu_model_runner.py
+2
-2
@@ -234,8 +234,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
234
if self.model_config.is_encoder_decoder:
235
# Maximum length of the encoder input, only for encoder-decoder
236
# models.
237
- self.max_encoder_len = self.mm_registry.\
238
- get_encdec_max_encoder_len(model_config)
237
+ self.max_encoder_len = scheduler_config.\
238
+ max_num_encoder_input_tokens
239
else:
240
self.max_encoder_len = 0
241