A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Update the warning log when using VLLM_ALLOW_LONG_MAX_MODEL_LEN (#20904)
Signed-off-by: wang.yuqi <noooop@126.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
wang.yuqi committed
Sep 1, 2025 at 16:50 UTC
55602bb2e695108e35501dfcd5890e7664c31495
1 file changed
+10
-6
vllm/config/__init__.py
+10
-6
@@ -3021,16 +3021,20 @@ def _get_and_verify_max_len(
3021
f"User-specified max_model_len ({max_model_len}) is greater "
3022
f"than the derived max_model_len ({max_len_key}="
3023
f"{derived_max_model_len} or model_max_length="
3024
- f"{model_max_length} in model's config.json). This may lead "
3025
- "to incorrect model outputs or CUDA errors.")
3024
+ f"{model_max_length} in model's config.json).")
3025
+ warning = (
3026
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN must be used with extreme "
3027
+ "caution. If the model uses relative position encoding (RoPE), "
3028
+ "positions exceeding derived_max_model_len lead to nan. If the "
3029
+ "model uses absolute position encoding, positions exceeding "
3030
+ "derived_max_model_len will cause a CUDA array out-of-bounds "
3031
+ "error.")
3032
if envs.VLLM_ALLOW_LONG_MAX_MODEL_LEN:
3027
- logger.warning(
3028
- "%s Make sure the value is correct and within the "
3029
- "model context size.", msg)
3033
+ logger.warning_once("%s %s", msg, warning)
3034
else:
3035
raise ValueError(
3036
f"{msg} To allow overriding this maximum, set "
3033
- "the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1")
3037
+ f"the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1. {warning}")
3038
return int(max_model_len)
3039
3040