[Config] Update default `_max_num_batched_tokens` from 8192 to 16384 (#51726)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Aug 11, 2026 at 11:09 UTC
4988df2eb80a8d1debae7de85c3b242fa436f3c1
2 files changed
+12
-31
tests/v1/engine/test_engine_args.py
-29
@@ -5,7 +5,6 @@ from argparse import ArgumentError
5
6
import pytest
7
8
-from vllm.config import VllmConfig
8
from vllm.engine.arg_utils import EngineArgs
9
from vllm.usage.usage_lib import UsageContext
10
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -64,34 +63,6 @@ def test_prefix_caching_xxhash_from_cli():
63
assert vllm_config.cache_config.prefix_caching_hash_algo == "xxhash_cbor"
64
65
67
-def test_defaults_with_usage_context():
68
- engine_args = EngineArgs(model="facebook/opt-125m")
69
- vllm_config: VllmConfig = engine_args.create_engine_config(UsageContext.LLM_CLASS)
70
-
71
- from vllm.platforms import current_platform
72
- from vllm.utils.mem_constants import GiB_bytes
73
-
74
- device_memory = current_platform.get_device_total_memory()
75
- device_name = current_platform.get_device_name().lower()
76
- if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
77
- # For GPUs like H100, H200, and MI300x with >= 70GB memory
78
- default_llm_tokens = 16384
79
- default_server_tokens = 8192
80
- default_max_num_seqs = 1024
81
- else:
82
- default_llm_tokens = 8192
83
- default_server_tokens = 2048
84
- default_max_num_seqs = 256
85
-
86
- assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs
87
- assert vllm_config.scheduler_config.max_num_batched_tokens == default_llm_tokens # noqa: E501
88
-
89
- engine_args = EngineArgs(model="facebook/opt-125m")
90
- vllm_config = engine_args.create_engine_config(UsageContext.OPENAI_API_SERVER)
91
- assert vllm_config.scheduler_config.max_num_seqs == default_max_num_seqs
92
- assert vllm_config.scheduler_config.max_num_batched_tokens == default_server_tokens # noqa: E501
93
-
94
-
66
def test_mm_prefix_lm_raises_batched_tokens_floor():
67
"""Verify that prefix-LM multimodal models auto-raise
68
max_num_batched_tokens to fit at least one multimodal item.
vllm/engine/arg_utils.py
+12
-2
@@ -2566,8 +2566,18 @@ class EngineArgs:
2566
# NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
2567
# throughput, see PR #17885 for more details.
2568
# So here we do an extra device name check to prevent such regression.
2569
- if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
2570
- # For GPUs like H100 and MI300x, use larger default values.
2569
+ if device_memory >= 160 * GiB_bytes:
2570
+ # for GPUs like B200/B300 with >= 160GB memory, use the largest defaults
2571
+ default_max_num_batched_tokens = {
2572
+ UsageContext.LLM_CLASS: 16384,
2573
+ UsageContext.OPENAI_API_SERVER: 16384,
2574
+ }
2575
+ default_max_num_seqs = {
2576
+ UsageContext.LLM_CLASS: 1024,
2577
+ UsageContext.OPENAI_API_SERVER: 1024,
2578
+ }
2579
+ elif device_memory >= 70 * GiB_bytes and "a100" not in device_name:
2580
+ # For GPUs like H100 and H200, use larger offline defaults.
2581
default_max_num_batched_tokens = {
2582
UsageContext.LLM_CLASS: 16384,
2583
UsageContext.OPENAI_API_SERVER: 8192,