A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Make `max_model_len` configurable (#972)
Antoni Baum committed
Sep 12, 2023 at 16:29 UTC
0bb1e885a08df59beec7149b1d0d646e24ab1a42
2 files changed
+22
-1
vllm/config.py
+15
@@ -38,6 +38,8 @@ class ModelConfig:
38
will use FP16 precision for FP32 and FP16 models, and BF16 precision
39
for BF16 models.
40
seed: Random seed for reproducibility.
41
+ max_model_len: Maximum length of a sequence (including prompt and
42
+ output). If None, will be derived from the model.
43
"""
44
45
def __init__(
@@ -50,6 +52,7 @@ class ModelConfig:
52
load_format: str,
53
dtype: str,
54
seed: int,
55
+ max_model_len: Optional[int] = None,
56
) -> None:
57
self.model = model
58
self.tokenizer = tokenizer
@@ -63,6 +66,16 @@ class ModelConfig:
66
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
67
self._verify_load_format()
68
self._verify_tokenizer_mode()
69
+ self.max_model_len = None
70
+ if max_model_len is not None:
71
+ derived_max_model_len = self.get_max_model_len()
72
+ if max_model_len > derived_max_model_len:
73
+ logger.warning(
74
+ f"User-specified max_model_len ({max_model_len}) is "
75
+ f"greater than the derived max_model_len "
76
+ f"({derived_max_model_len}). Make sure the value is "
77
+ "correct and within the model context size.")
78
+ self.max_model_len = max_model_len
79
80
def _verify_load_format(self) -> None:
81
load_format = self.load_format.lower()
@@ -134,6 +147,8 @@ class ModelConfig:
147
return total_num_attention_heads // parallel_config.tensor_parallel_size
148
149
def get_max_model_len(self) -> int:
150
+ if self.max_model_len is not None:
151
+ return self.max_model_len
152
max_model_len = float("inf")
153
possible_keys = [
154
# OPT
vllm/engine/arg_utils.py
+7
-1
@@ -18,6 +18,7 @@ class EngineArgs:
18
load_format: str = 'auto'
19
dtype: str = 'auto'
20
seed: int = 0
21
+ max_model_len: Optional[int] = None
22
worker_use_ray: bool = False
23
pipeline_parallel_size: int = 1
24
tensor_parallel_size: int = 1
@@ -89,6 +90,11 @@ class EngineArgs:
90
'The "auto" option will use FP16 precision '
91
'for FP32 and FP16 models, and BF16 precision '
92
'for BF16 models.')
93
+ parser.add_argument('--max-model-len',
94
+ type=int,
95
+ default=None,
96
+ help='model context length. If unspecified, '
97
+ 'will be automatically derived from the model.')
98
# Parallel arguments
99
parser.add_argument('--worker-use-ray',
100
action='store_true',
@@ -153,7 +159,7 @@ class EngineArgs:
159
model_config = ModelConfig(self.model, self.tokenizer,
160
self.tokenizer_mode, self.trust_remote_code,
161
self.download_dir, self.load_format,
156
- self.dtype, self.seed)
162
+ self.dtype, self.seed, self.max_model_len)
163
cache_config = CacheConfig(self.block_size,
164
self.gpu_memory_utilization,
165
self.swap_space)