A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Automatically configure `max_num_batched_tokens` (#1198)
Woosuk Kwon committed
Sep 27, 2023 at 16:34 UTC
a19bc5c6281cb1d539043acb06699bf8438bb254
2 files changed
+35
-11
vllm/config.py
+34
-9
@@ -266,11 +266,36 @@ class SchedulerConfig:
266
and generated text).
267
"""
268
269
- def __init__(self, max_num_batched_tokens: int, max_num_seqs: int,
270
- max_model_len: int) -> None:
271
- self.max_num_batched_tokens = max_num_batched_tokens
269
+ def __init__(
270
+ self,
271
+ max_num_batched_tokens: Optional[int],
272
+ max_num_seqs: int,
273
+ max_model_len: int,
274
+ ) -> None:
275
+ if max_num_batched_tokens is not None:
276
+ self.max_num_batched_tokens = max_num_batched_tokens
277
+ else:
278
+ # If max_model_len is too short, use 2048 as the default value for
279
+ # higher throughput.
280
+ self.max_num_batched_tokens = max(max_model_len, 2048)
281
self.max_num_seqs = max_num_seqs
282
self.max_model_len = max_model_len
283
+ self._verify_args()
284
+
285
+ def _verify_args(self) -> None:
286
+ if self.max_num_batched_tokens < self.max_model_len:
287
+ raise ValueError(
288
+ f"max_num_batched_tokens ({self.max_num_batched_tokens}) is "
289
+ f"smaller than max_model_len ({self.max_model_len}). "
290
+ "This effectively limits the maximum sequence length to "
291
+ "max_num_batched_tokens and makes vLLM reject longer "
292
+ "sequences. Please increase max_num_batched_tokens or "
293
+ "decrease max_model_len.")
294
+ if self.max_num_batched_tokens < self.max_num_seqs:
295
+ raise ValueError(
296
+ f"max_num_batched_tokens ({self.max_num_batched_tokens}) must "
297
+ "be greater than or equal to max_num_seqs "
298
+ f"({self.max_num_seqs}).")
299
300
301
_STR_DTYPE_TO_TORCH_DTYPE = {
@@ -350,14 +375,14 @@ def _get_and_verify_max_len(
375
max_len_key = getattr(hf_config, key, None)
376
if max_len_key is not None:
377
derived_max_model_len = min(derived_max_model_len, max_len_key)
378
+ if derived_max_model_len == float("inf"):
379
+ raise ValueError(
380
+ "The model's config.json must contain one of the following keys "
381
+ "to determine the original maximum length of the model: "
382
+ f"{possible_keys}")
383
384
rope_scaling = getattr(hf_config, "rope_scaling", None)
385
if rope_scaling is not None:
356
- if derived_max_model_len == float("inf"):
357
- raise ValueError(
358
- "When using rope_scaling, the model's config.json must "
359
- "contain one of the following keys to determine the original "
360
- f"maximum length of the model: {possible_keys}")
386
assert "factor" in rope_scaling
387
scaling_factor = rope_scaling["factor"]
388
derived_max_model_len *= scaling_factor
@@ -371,4 +396,4 @@ def _get_and_verify_max_len(
396
" in model's config.json). This may lead to incorrect model "
397
"outputs or CUDA errors. Make sure the value is correct and "
398
"within the model context size.")
374
- return max_model_len
399
+ return int(max_model_len)
vllm/engine/arg_utils.py
+1
-2
@@ -25,7 +25,7 @@ class EngineArgs:
25
block_size: int = 16
26
swap_space: int = 4 # GiB
27
gpu_memory_utilization: float = 0.90
28
- max_num_batched_tokens: int = 2560
28
+ max_num_batched_tokens: Optional[int] = None
29
max_num_seqs: int = 256
30
disable_log_stats: bool = False
31
revision: Optional[str] = None
@@ -34,7 +34,6 @@ class EngineArgs:
34
def __post_init__(self):
35
if self.tokenizer is None:
36
self.tokenizer = self.model
37
- self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)
37
38
@staticmethod
39
def add_cli_args(