A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Fix Whisper crash caused by invalid``` max_num_batched_tokens``` config (#17853)
Signed-off-by: inkcherry <mingzhi.liu@intel.com>
inkcherry committed
May 9, 2025 at 17:16 UTC
5b2dcbf0b8dd9ee9199d7496c84e84c010122a00
1 file changed
+14
vllm/config.py
+14
@@ -2050,6 +2050,13 @@ class SchedulerConfig:
2050
_MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
2051
)
2052
2053
+ # When using default settings,
2054
+ # Ensure max_num_batched_tokens does not exceed model limit.
2055
+ # Some models (e.g., Whisper) have embeddings tied to max length.
2056
+ self.max_num_batched_tokens = min(
2057
+ self.max_num_seqs * self.max_model_len,
2058
+ self.max_num_batched_tokens)
2059
+
2060
self.max_num_encoder_input_tokens = self.max_num_batched_tokens
2061
self.encoder_cache_size = self.max_num_batched_tokens
2062
@@ -2090,6 +2097,13 @@ class SchedulerConfig:
2097
"be greater than or equal to max_num_seqs "
2098
f"({self.max_num_seqs}).")
2099
2100
+ if self.max_num_batched_tokens > self.max_num_seqs * self.max_model_len:
2101
+ logger.warning(
2102
+ "max_num_batched_tokens (%d) exceeds max_num_seqs"
2103
+ "* max_model_len (%d). This may lead to unexpected behavior.",
2104
+ self.max_num_batched_tokens,
2105
+ self.max_num_seqs * self.max_model_len)
2106
+
2107
if self.num_lookahead_slots < 0:
2108
raise ValueError(
2109
"num_lookahead_slots "