A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[DP] Support api-server-count > 0 in hybrid DP LB mode (#21510)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jul 25, 2025 at 04:18 UTC
9c8b2c2a8a0bbfa0c43ed7b220642dc3e802633c
2 files changed
+5
-9
tests/v1/test_hybrid_lb_dp.py
+1
-1
@@ -147,7 +147,7 @@ def default_server_args():
147
]
148
149
150
-@pytest.fixture(scope="module", params=[1]) # Only 1 API server for now
150
+@pytest.fixture(scope="module", params=[1, 4])
151
def servers(request, default_server_args):
152
api_server_count = request.param
153
with HybridLBServerManager(MODEL_NAME, DP_SIZE, api_server_count,
vllm/entrypoints/cli/serve.py
+4
-8
@@ -165,18 +165,14 @@ def run_multi_api_server(args: argparse.Namespace):
165
" api_server_count > 1")
166
model_config.disable_mm_preprocessor_cache = True
167
168
- if vllm_config.parallel_config.data_parallel_hybrid_lb:
169
- raise NotImplementedError(
170
- "Hybrid load balancing with --api-server-count > 0"
171
- "is not yet supported.")
172
-
168
executor_class = Executor.get_class(vllm_config)
169
log_stats = not engine_args.disable_log_stats
170
171
parallel_config = vllm_config.parallel_config
172
dp_rank = parallel_config.data_parallel_rank
173
external_dp_lb = parallel_config.data_parallel_external_lb
179
- assert external_dp_lb or dp_rank == 0
174
+ hybrid_dp_lb = parallel_config.data_parallel_hybrid_lb
175
+ assert external_dp_lb or hybrid_dp_lb or dp_rank == 0
176
177
api_server_manager: Optional[APIServerProcessManager] = None
178
@@ -196,12 +192,12 @@ def run_multi_api_server(args: argparse.Namespace):
192
stats_update_address=coordinator.get_stats_publish_address()
193
if coordinator else None)
194
199
- # For dp ranks > 0 in external DP LB mode, we must delay the
195
+ # For dp ranks > 0 in external/hybrid DP LB modes, we must delay the
196
# start of the API servers until the local engine is started
197
# (after the launcher context manager exits),
198
# since we get the front-end stats update address from the coordinator
199
# via the handshake with the local engine.
204
- if dp_rank == 0 or not external_dp_lb:
200
+ if dp_rank == 0 or not (external_dp_lb or hybrid_dp_lb):
201
# Start API servers using the manager.
202
api_server_manager = APIServerProcessManager(
203
**api_server_manager_kwargs)