@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 3d99b0499a

[Logs] DP Supervisor Log Improvement (#48278)

Signed-off-by: Robert Shaw <robertgshaw2-redhat@ip-172-31-18-125.us-east-2.compute.internal> Co-authored-by: Robert Shaw <robertgshaw2-redhat@ip-172-31-18-125.us-east-2.compute.internal>

Robert Shaw committed Jul 11, 2026 at 00:07 UTC 3d99b0499aff7ce3b91942b0385558dab4eecf76

2 files changed +96 -42

tests/entrypoints/openai/test_dp_supervisor.py

+50 -30

@@ -301,9 +301,20 @@ async def test_shutdown_if_supervisor_server_error_on_startup(

301 async def fake_shutdown_children(self):

302 return None

303

304 + def fake_start_children(self):

305 + return None

306 +

307 + async def fake_monitor_children(self):

308 + # Mark ready so the supervisor server is started, then block until

309 + # shutdown (triggered when the failing server task exits).

310 + self._is_ready = True

311 + await self._shutdown_event.wait()

312 +

313 monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop())

314 monkeypatch.setattr(dp_sup.uvicorn, "Server", FakeServer)

315 monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children)

316 + monkeypatch.setattr(DPSupervisor, "_start_children", fake_start_children)

317 + monkeypatch.setattr(DPSupervisor, "_monitor_children", fake_monitor_children)

318

319 supervisor = DPSupervisor(_make_unit_args())

320

@@ -437,8 +448,11 @@ def launch_mock_vllm_with_drain(

448

449 async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -> bool:

450 """

440 - Poll GET /health on the supervisor until expected_status is seen.

441 - A connection error is treated as 503-equivalent when expected_status != 200.

451 + GET /health on the supervisor once and check for expected_status.

452 +

453 + Pass expected_status=-1 to assert the supervisor is not listening yet

454 + (a connection error is expected). The supervisor only starts its HTTP

455 + server once every child is ready, so /health is refused until then.

456 """

457 scheme = "https" if use_ssl else "http"

458 url = f"{scheme}://127.0.0.1:{_SUPERVISOR_PORT}/health"

@@ -456,6 +470,17 @@ async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -

470 return True

471

472

473 +async def _await_supervisor_health(

474 + expected_status: int, use_ssl: bool = False, retries: int = 20

475 +) -> bool:

476 + """Retry _poll_supervisor_health, tolerating supervisor server startup."""

477 + for _ in range(retries):

478 + if await _poll_supervisor_health(expected_status, use_ssl=use_ssl):

479 + return True

480 + await asyncio.sleep(0.5)

481 + return False

482 +

483 +

484 async def _poll_until_api_server_running(

485 port: int, retries: int = 10, use_ssl: bool = False

486 ) -> None:

@@ -534,7 +559,7 @@ async def _run_supervisor(

559 @pytest.mark.asyncio

560 async def test_basic_lifecycle(monkeypatch):

561 """

537 - A) Supervisor /health returns 503 while children are unhealthy.

562 + A) Supervisor is not listening while children are unhealthy.

563 B) /health returns 200 once every child reports healthy.

564 C) SIGTERM and shutdown

565 """

@@ -543,24 +568,23 @@ async def test_basic_lifecycle(monkeypatch):

568 vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]

569

570 async with _run_supervisor(args, monkeypatch) as (supervisor, _task):

546 - assert await _poll_supervisor_health(503)

571 + assert await _poll_supervisor_health(-1)

572 assert not supervisor.is_ready

573

574 for port in vllm_server_ports:

550 - assert await _poll_supervisor_health(503)

575 + assert await _poll_supervisor_health(-1)

576 assert not supervisor.is_ready

577 await _poll_until_api_server_running(port)

578

579 await _set_healthy(vllm_server_ports[0])

580 await asyncio.sleep(1.0)

556 - assert await _poll_supervisor_health(503)

581 + assert await _poll_supervisor_health(-1)

582 assert not supervisor.is_ready

558 - print("/health is 503 --- expected!")

583 + print("supervisor not listening --- expected!")

584

585 for port in vllm_server_ports:

586 await _set_healthy(port)

562 - await asyncio.sleep(1.0)

563 - assert await _poll_supervisor_health(200)

587 + assert await _await_supervisor_health(200)

588 assert supervisor.is_ready

589 print("/health is 200 --- expected!")

590

@@ -589,19 +613,18 @@ async def test_basic_lifecycle_with_ssl(monkeypatch):

613 vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]

614

615 async with _run_supervisor(args, monkeypatch) as (supervisor, _task):

592 - assert await _poll_supervisor_health(503, use_ssl=True)

616 + assert await _poll_supervisor_health(-1, use_ssl=True)

617 assert not supervisor.is_ready

618

619 for port in vllm_server_ports:

596 - assert await _poll_supervisor_health(503, use_ssl=True)

620 + assert await _poll_supervisor_health(-1, use_ssl=True)

621 assert not supervisor.is_ready

622 await _poll_until_api_server_running(port, use_ssl=True)

623

624 for port in vllm_server_ports:

625 await _set_healthy(port, use_ssl=True)

602 - await asyncio.sleep(1.0)

626

604 - assert await _poll_supervisor_health(200, use_ssl=True)

627 + assert await _await_supervisor_health(200, use_ssl=True)

628 assert supervisor.is_ready

629

630

@@ -616,7 +639,7 @@ async def test_failed_startup(monkeypatch):

639 vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]

640

641 async with _run_supervisor(args, monkeypatch) as (supervisor, _task):

619 - assert await _poll_supervisor_health(503)

642 + assert await _poll_supervisor_health(-1)

643 assert not supervisor.is_ready

644

645 for port in vllm_server_ports:

@@ -632,7 +655,7 @@ async def test_failed_startup(monkeypatch):

655 @pytest.mark.asyncio

656 async def test_becomes_unhealthy(monkeypatch):

657 """

635 - A) Supervisor /health returns 503 while children are unhealthy.

658 + A) Supervisor is not listening while children are unhealthy.

659 B) /health returns 200 once every child reports healthy.

660 C) Child process becomes unhealthy.

661 D) Detected and shutdown.

@@ -642,24 +665,23 @@ async def test_becomes_unhealthy(monkeypatch):

665 vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]

666

667 async with _run_supervisor(args, monkeypatch) as (supervisor, _task):

645 - assert await _poll_supervisor_health(503)

668 + assert await _poll_supervisor_health(-1)

669 assert not supervisor.is_ready

670

671 for port in vllm_server_ports:

649 - assert await _poll_supervisor_health(503)

672 + assert await _poll_supervisor_health(-1)

673 assert not supervisor.is_ready

674 await _poll_until_api_server_running(port)

675

676 await _set_healthy(vllm_server_ports[0])

677 await asyncio.sleep(1.0)

655 - assert await _poll_supervisor_health(503)

678 + assert await _poll_supervisor_health(-1)

679 assert not supervisor.is_ready

657 - print("/health is 503 --- expected!")

680 + print("supervisor not listening --- expected!")

681

682 for port in vllm_server_ports:

683 await _set_healthy(port)

661 - await asyncio.sleep(1.0)

662 - assert await _poll_supervisor_health(200)

684 + assert await _await_supervisor_health(200)

685 assert supervisor.is_ready

686 print("/health is 200 --- expected!")

687

@@ -674,7 +696,7 @@ async def test_becomes_unhealthy(monkeypatch):

696 @pytest.mark.asyncio

697 async def test_dp_server_fails(monkeypatch):

698 """

677 - A) Supervisor /health returns 503 while children are unhealthy.

699 + A) Supervisor is not listening while children are unhealthy.

700 B) /health returns 200 once every child reports healthy.

701 C) Child process fails.

702 D) Detected and shutdown.

@@ -684,24 +706,23 @@ async def test_dp_server_fails(monkeypatch):

706 vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]

707

708 async with _run_supervisor(args, monkeypatch) as (supervisor, _task):

687 - assert await _poll_supervisor_health(503)

709 + assert await _poll_supervisor_health(-1)

710 assert not supervisor.is_ready

711

712 for port in vllm_server_ports:

691 - assert await _poll_supervisor_health(503)

713 + assert await _poll_supervisor_health(-1)

714 assert not supervisor.is_ready

715 await _poll_until_api_server_running(port)

716

717 await _set_healthy(vllm_server_ports[0])

718 await asyncio.sleep(1.0)

697 - assert await _poll_supervisor_health(503)

719 + assert await _poll_supervisor_health(-1)

720 assert not supervisor.is_ready

699 - print("/health is 503 --- expected!")

721 + print("supervisor not listening --- expected!")

722

723 for port in vllm_server_ports:

724 await _set_healthy(port)

703 - await asyncio.sleep(1.0)

704 - assert await _poll_supervisor_health(200)

725 + assert await _await_supervisor_health(200)

726 assert supervisor.is_ready

727 print("/health is 200 --- expected!")

728

@@ -739,8 +760,7 @@ async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch):

760

761 for port in vllm_server_ports:

762 await _set_healthy(port)

742 - await asyncio.sleep(1.0)

743 - assert await _poll_supervisor_health(200)

763 + assert await _await_supervisor_health(200)

764 assert supervisor.is_ready

765

766 start_t = time.perf_counter()

vllm/entrypoints/openai/dp_supervisor.py

+46 -12

@@ -283,14 +283,46 @@ class DPSupervisor:

283

284 async def run(self) -> None:

285 loop = asyncio.get_running_loop()

286 + decorate_logs("DPSupervisor")

287

288 # K8s sends SIGTERM for shutdown - begin graceful termination.

289 for sig in (signal.SIGTERM, signal.SIGINT):

290 loop.add_signal_handler(sig, partial(self._handle_signal, sig))

291

291 - # Launch DPSupervisor Server.

292 + supervisor_server: uvicorn.Server | None = None

293 + supervisor_server_task: asyncio.Task[None] | None = None

294 + try:

295 + # Launch vLLM DP Servers and begin monitoring them.

296 + self._start_children()

297 + monitor_task = asyncio.create_task(

298 + self._monitor_children(), name="dp-monitor"

299 + )

300 +

301 + # Only start the DPSupervisor server once the vLLM DP Servers are

302 + # ready. This avoids the supervisor /health endpoint returning 503

303 + # to external load balancer probes while the engines initialize.

304 + await self._wait_until_ready(monitor_task)

305 + if self.is_ready and not monitor_task.done():

306 + supervisor_server, supervisor_server_task = await self._start_server()

307 +

308 + await monitor_task

309 + finally:

310 + self._is_ready = False

311 + await self._shutdown_children()

312 +

313 + # Shutdown the DP Supervisor server.

314 + if supervisor_server is not None and supervisor_server_task is not None:

315 + supervisor_server.should_exit = True

316 + await supervisor_server_task

317 +

318 + async def _start_server(self) -> tuple[uvicorn.Server, asyncio.Task[None]]:

319 + """

320 + Launch the DPSupervisor HTTP server.

321 +

322 + Called only after the vLLM DP Servers are ready so that /health does

323 + not return 503 to external probes while the engines are initializing.

324 + """

325 app = _build_dp_supervisor_app(self)

293 - decorate_logs("DPSupervisor")

326 host = self.args.host or "0.0.0.0"

327 config = uvicorn.Config(

328 app,

@@ -319,18 +351,20 @@ class DPSupervisor:

351 raise RuntimeError("DPSupervisor exited before startup.")

352 await asyncio.sleep(0.05)

353 logger.info("Started DPSupervisor on %s:%d", host, self.supervisor_port)

354 + return supervisor_server, supervisor_server_task

355

323 - # Launch and Monitor vLLM Server Processes.

324 - try:

325 - self._start_children()

326 - await self._monitor_children()

327 - finally:

328 - self._is_ready = False

329 - await self._shutdown_children()

356 + async def _wait_until_ready(self, monitor_task: asyncio.Task[None]) -> None:

357 + """

358 + Block until the vLLM DP Servers are ready or shutdown is triggered.

359

331 - # Shutdown the DP Supervisor server.

332 - supervisor_server.should_exit = True

333 - await supervisor_server_task

360 + Returns early if monitoring stops (e.g. a DP Server dies during

361 + startup), in which case the supervisor server is never started.

362 + """

363 + logger.info("Waiting for vLLM DP Servers to become ready.")

364 + while not self._is_ready and not self._shutdown_event.is_set():

365 + if monitor_task.done():

366 + return

367 + await asyncio.sleep(0.05)

368

369 def _handle_signal(self, signum: int) -> None:

370 """