A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 18:15 UTC
[UX] Default api_server_count to dp_size if not specified (#32525)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Tyler Michael Smith committed
Jan 22, 2026 at 12:35 UTC
803e3f3f688677701a616d692125e181116d6b69
2 files changed
+60
-8
vllm/entrypoints/cli/serve.py
+57
-6
@@ -50,14 +50,65 @@ class ServeSubcommand(CLISubcommand):
50
if hasattr(args, "model_tag") and args.model_tag is not None:
51
args.model = args.model_tag
52
53
- if args.headless or args.api_server_count < 1:
53
+ if args.headless:
54
+ if args.api_server_count is not None and args.api_server_count > 0:
55
+ raise ValueError(
56
+ f"--api-server-count={args.api_server_count} cannot be "
57
+ "used with --headless (no API servers are started in "
58
+ "headless mode)."
59
+ )
60
+ # Default to 0 in headless mode (no API servers)
61
+ args.api_server_count = 0
62
+
63
+ # Detect LB mode for defaulting api_server_count.
64
+ # External LB: --data-parallel-external-lb or --data-parallel-rank
65
+ # Hybrid LB: --data-parallel-hybrid-lb or --data-parallel-start-rank
66
+ is_external_lb = (
67
+ args.data_parallel_external_lb or args.data_parallel_rank is not None
68
+ )
69
+ is_hybrid_lb = (
70
+ args.data_parallel_hybrid_lb or args.data_parallel_start_rank is not None
71
+ )
72
+
73
+ if is_external_lb and is_hybrid_lb:
74
+ raise ValueError(
75
+ "Cannot use both external and hybrid data parallel load "
76
+ "balancing modes. External LB is enabled via "
77
+ "--data-parallel-external-lb or --data-parallel-rank. "
78
+ "Hybrid LB is enabled via --data-parallel-hybrid-lb or "
79
+ "--data-parallel-start-rank. Use one mode or the other."
80
+ )
81
+
82
+ # Default api_server_count if not explicitly set.
83
+ # - External LB: Leave as 1 (external LB handles distribution)
84
+ # - Hybrid LB: Use local DP size (internal LB for local ranks only)
85
+ # - Internal LB: Use full DP size
86
+ if args.api_server_count is None:
87
+ if is_external_lb:
88
+ args.api_server_count = 1
89
+ elif is_hybrid_lb:
90
+ args.api_server_count = args.data_parallel_size_local or 1
91
+ if args.api_server_count > 1:
92
+ logger.info(
93
+ "Defaulting api_server_count to data_parallel_size_local "
94
+ "(%d) for hybrid LB mode.",
95
+ args.api_server_count,
96
+ )
97
+ else:
98
+ args.api_server_count = args.data_parallel_size
99
+ if args.api_server_count > 1:
100
+ logger.info(
101
+ "Defaulting api_server_count to data_parallel_size (%d).",
102
+ args.api_server_count,
103
+ )
104
+
105
+ if args.api_server_count < 1:
106
run_headless(args)
107
+ elif args.api_server_count > 1:
108
+ run_multi_api_server(args)
109
else:
56
- if args.api_server_count > 1:
57
- run_multi_api_server(args)
58
- else:
59
- # Single API server (this process).
60
- uvloop.run(run_server(args))
110
+ # Single API server (this process).
111
+ uvloop.run(run_server(args))
112
113
def validate(self, args: argparse.Namespace) -> None:
114
validate_parsed_serve_args(args)
vllm/entrypoints/openai/cli_args.py
+3
-2
@@ -283,8 +283,9 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
283
"--api-server-count",
284
"-asc",
285
type=int,
286
- default=1,
287
- help="How many API server processes to run.",
286
+ default=None,
287
+ help="How many API server processes to run. "
288
+ "Defaults to data_parallel_size if not specified.",
289
)
290
parser.add_argument(
291
"--config",