@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0bb1e885a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Make `max_model_len` configurable (#972)

Antoni Baum committed Sep 12, 2023 at 16:29 UTC 0bb1e885a08df59beec7149b1d0d646e24ab1a42
2 files changed +22 -1
vllm/config.py
+15
@@ -38,6 +38,8 @@ class ModelConfig:
38 will use FP16 precision for FP32 and FP16 models, and BF16 precision
39 for BF16 models.
40 seed: Random seed for reproducibility.
41 + max_model_len: Maximum length of a sequence (including prompt and
42 + output). If None, will be derived from the model.
43 """
44
45 def __init__(
@@ -50,6 +52,7 @@ class ModelConfig:
52 load_format: str,
53 dtype: str,
54 seed: int,
55 + max_model_len: Optional[int] = None,
56 ) -> None:
57 self.model = model
58 self.tokenizer = tokenizer
@@ -63,6 +66,16 @@ class ModelConfig:
66 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
67 self._verify_load_format()
68 self._verify_tokenizer_mode()
69 + self.max_model_len = None
70 + if max_model_len is not None:
71 + derived_max_model_len = self.get_max_model_len()
72 + if max_model_len > derived_max_model_len:
73 + logger.warning(
74 + f"User-specified max_model_len ({max_model_len}) is "
75 + f"greater than the derived max_model_len "
76 + f"({derived_max_model_len}). Make sure the value is "
77 + "correct and within the model context size.")
78 + self.max_model_len = max_model_len
79
80 def _verify_load_format(self) -> None:
81 load_format = self.load_format.lower()
@@ -134,6 +147,8 @@ class ModelConfig:
147 return total_num_attention_heads // parallel_config.tensor_parallel_size
148
149 def get_max_model_len(self) -> int:
150 + if self.max_model_len is not None:
151 + return self.max_model_len
152 max_model_len = float("inf")
153 possible_keys = [
154 # OPT
vllm/engine/arg_utils.py
+7 -1
@@ -18,6 +18,7 @@ class EngineArgs:
18 load_format: str = 'auto'
19 dtype: str = 'auto'
20 seed: int = 0
21 + max_model_len: Optional[int] = None
22 worker_use_ray: bool = False
23 pipeline_parallel_size: int = 1
24 tensor_parallel_size: int = 1
@@ -89,6 +90,11 @@ class EngineArgs:
90 'The "auto" option will use FP16 precision '
91 'for FP32 and FP16 models, and BF16 precision '
92 'for BF16 models.')
93 + parser.add_argument('--max-model-len',
94 + type=int,
95 + default=None,
96 + help='model context length. If unspecified, '
97 + 'will be automatically derived from the model.')
98 # Parallel arguments
99 parser.add_argument('--worker-use-ray',
100 action='store_true',
@@ -153,7 +159,7 @@ class EngineArgs:
159 model_config = ModelConfig(self.model, self.tokenizer,
160 self.tokenizer_mode, self.trust_remote_code,
161 self.download_dir, self.load_format,
156 - self.dtype, self.seed)
162 + self.dtype, self.seed, self.max_model_len)
163 cache_config = CacheConfig(self.block_size,
164 self.gpu_memory_utilization,
165 self.swap_space)