A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] Warn if user `max_model_len` is greater than derived `max_model_len` (#7080)
Signed-off-by: Jefferson Fialho <jfialho@ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Jeff Fialho committed
Aug 3, 2024 at 20:01 UTC
825b044863a8e3af82a82a80cd2617486cc829ca
2 files changed
+23
-6
vllm/config.py
+13
-6
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Type, Union
6
import torch
7
from transformers import PretrainedConfig
8
9
+import vllm.envs as envs
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.model_executor.models import ModelRegistry
@@ -1541,15 +1542,21 @@ def _get_and_verify_max_len(
1542
"Disabling sliding window is not supported for models "
1543
"model_max_length in the config. Please raise an issue "
1544
"so we can investigate.")
1544
- pass
1545
else:
1546
- raise ValueError(
1546
+ msg = (
1547
f"User-specified max_model_len ({max_model_len}) is greater "
1548
- "than the derived max_model_len "
1549
- f"({max_len_key}={derived_max_model_len} or model_max_length="
1548
+ f"than the derived max_model_len ({max_len_key}="
1549
+ f"{derived_max_model_len} or model_max_length="
1550
f"{model_max_length} in model's config.json). This may lead "
1551
- "to incorrect model outputs or CUDA errors. Make sure the "
1552
- "value is correct and within the model context size.")
1551
+ "to incorrect model outputs or CUDA errors.")
1552
+ if envs.VLLM_ALLOW_LONG_MAX_MODEL_LEN:
1553
+ logger.warning(
1554
+ "%s Make sure the value is correct and within the "
1555
+ "model context size.", msg)
1556
+ else:
1557
+ raise ValueError(
1558
+ f"{msg} To allow overriding this maximum, set "
1559
+ "the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1")
1560
return int(max_model_len)
1561
1562
vllm/envs.py
+10
@@ -50,6 +50,7 @@ if TYPE_CHECKING:
50
VLLM_NO_DEPRECATION_WARNING: bool = False
51
CMAKE_BUILD_TYPE: Optional[str] = None
52
VERBOSE: bool = False
53
+ VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
54
55
56
def get_default_cache_root():
@@ -331,6 +332,15 @@ environment_variables: Dict[str, Callable[[], Any]] = {
332
# If set, vllm will skip the deprecation warnings.
333
"VLLM_NO_DEPRECATION_WARNING":
334
lambda: bool(int(os.getenv("VLLM_NO_DEPRECATION_WARNING", "0"))),
335
+
336
+ # If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows
337
+ # the user to specify a max sequence length greater than
338
+ # the max length derived from the model's config.json.
339
+ # To enable this, set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1.
340
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN":
341
+ lambda:
342
+ (os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower() in
343
+ ("1", "true")),
344
}
345
346
# end-env-vars-definition