@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 825b044863
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend] Warn if user `max_model_len` is greater than derived `max_model_len` (#7080)

Signed-off-by: Jefferson Fialho <jfialho@ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>

Jeff Fialho committed Aug 3, 2024 at 20:01 UTC 825b044863a8e3af82a82a80cd2617486cc829ca
2 files changed +23 -6
vllm/config.py
+13 -6
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Type, Union
6 import torch
7 from transformers import PretrainedConfig
8
9 +import vllm.envs as envs
10 from vllm.logger import init_logger
11 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12 from vllm.model_executor.models import ModelRegistry
@@ -1541,15 +1542,21 @@ def _get_and_verify_max_len(
1542 "Disabling sliding window is not supported for models "
1543 "model_max_length in the config. Please raise an issue "
1544 "so we can investigate.")
1544 - pass
1545 else:
1546 - raise ValueError(
1546 + msg = (
1547 f"User-specified max_model_len ({max_model_len}) is greater "
1548 - "than the derived max_model_len "
1549 - f"({max_len_key}={derived_max_model_len} or model_max_length="
1548 + f"than the derived max_model_len ({max_len_key}="
1549 + f"{derived_max_model_len} or model_max_length="
1550 f"{model_max_length} in model's config.json). This may lead "
1551 - "to incorrect model outputs or CUDA errors. Make sure the "
1552 - "value is correct and within the model context size.")
1551 + "to incorrect model outputs or CUDA errors.")
1552 + if envs.VLLM_ALLOW_LONG_MAX_MODEL_LEN:
1553 + logger.warning(
1554 + "%s Make sure the value is correct and within the "
1555 + "model context size.", msg)
1556 + else:
1557 + raise ValueError(
1558 + f"{msg} To allow overriding this maximum, set "
1559 + "the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1")
1560 return int(max_model_len)
1561
1562
vllm/envs.py
+10
@@ -50,6 +50,7 @@ if TYPE_CHECKING:
50 VLLM_NO_DEPRECATION_WARNING: bool = False
51 CMAKE_BUILD_TYPE: Optional[str] = None
52 VERBOSE: bool = False
53 + VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
54
55
56 def get_default_cache_root():
@@ -331,6 +332,15 @@ environment_variables: Dict[str, Callable[[], Any]] = {
332 # If set, vllm will skip the deprecation warnings.
333 "VLLM_NO_DEPRECATION_WARNING":
334 lambda: bool(int(os.getenv("VLLM_NO_DEPRECATION_WARNING", "0"))),
335 +
336 + # If the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN is set, it allows
337 + # the user to specify a max sequence length greater than
338 + # the max length derived from the model's config.json.
339 + # To enable this, set VLLM_ALLOW_LONG_MAX_MODEL_LEN=1.
340 + "VLLM_ALLOW_LONG_MAX_MODEL_LEN":
341 + lambda:
342 + (os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower() in
343 + ("1", "true")),
344 }
345
346 # end-env-vars-definition