A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Limit the default value of `max_model_len` when it is not specified by users (#27556)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Oct 27, 2025 at 18:16 UTC
a3e8611da5744b1f64f3c4be063bf4a7aed952f0
3 files changed
+28
-12
vllm/config/model.py
+5
-12
@@ -2112,20 +2112,13 @@ def _get_and_verify_max_len(
2112
if encoder_config and "max_seq_length" in encoder_config:
2113
derived_max_model_len = encoder_config["max_seq_length"]
2114
2115
- # If the user specified a max length, make sure it is smaller than the
2116
- # derived length from the HF model config.
2115
+ # If the user didn't specify `max_model_len`, then use that derived from
2116
+ # the model config as a default value.
2117
if max_model_len is None:
2118
max_model_len = int(derived_max_model_len)
2119
- if current_platform.is_tpu():
2120
- logger.warning(
2121
- "--max-model-len is not specified, "
2122
- "it's currently using model's default length %s, "
2123
- "which might be too large."
2124
- "Please input with --max-model-len based on your "
2125
- "request input length and output length, to avoid "
2126
- "unnecessary degradation.",
2127
- max_model_len,
2128
- )
2119
+ max_model_len = current_platform.check_max_model_len(max_model_len)
2120
+ # If the user specified a max length, make sure it is smaller than the
2121
+ # derived length from the HF model config.
2122
elif max_model_len > derived_max_model_len:
2123
# Some models might have a separate key for specifying model_max_length
2124
# that will be bigger than derived_max_model_len. We compare user input
vllm/platforms/interface.py
+7
@@ -608,6 +608,13 @@ class Platform:
608
"""
609
return None
610
611
+ @classmethod
612
+ def check_max_model_len(cls, max_model_len: int) -> int:
613
+ """
614
+ Check max_model_len for the current platform.
615
+ """
616
+ return max_model_len
617
+
618
619
class UnspecifiedPlatform(Platform):
620
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/tpu.py
+16
@@ -251,6 +251,22 @@ class TpuPlatform(Platform):
251
def use_sync_weight_loader(cls) -> bool:
252
return True
253
254
+ @classmethod
255
+ def check_max_model_len(cls, max_model_len: int) -> int:
256
+ """
257
+ Check max_model_len for the current platform.
258
+ """
259
+ logger.warning(
260
+ "--max-model-len is not specified, "
261
+ "it's currently using model's default length %d, "
262
+ "which might be too large."
263
+ "Please input with --max-model-len based on your "
264
+ "request input length and output length, to avoid "
265
+ "unnecessary degradation.",
266
+ max_model_len,
267
+ )
268
+ return max_model_len
269
+
270
271
try:
272
from tpu_inference.platforms import TpuPlatform as TpuInferencePlatform