A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 09:45 UTC
[Feature] SSL support for dp supervisor (#43688)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
May 29, 2026 at 15:28 UTC
5dbf1605a0e2e4927533ac4b279edaf753580173
2 files changed
+143
-22
tests/entrypoints/openai/test_dp_supervisor.py
+125
-18
@@ -21,7 +21,10 @@ import asyncio
21
import contextlib
22
import os
23
import signal
24
+import subprocess
25
+import tempfile
26
import time
27
+from pathlib import Path
28
from types import SimpleNamespace
29
30
import aiohttp
@@ -35,6 +38,7 @@ from vllm.entrypoints.openai.dp_supervisor import (
38
DPSupervisor,
39
_build_vllm_dp_server_args,
40
infer_multi_port_external_lb_start_rank,
41
+ validate_multi_port_external_lb_args,
42
)
43
from vllm.logger import init_logger
44
@@ -75,6 +79,8 @@ def _make_unit_args(**overrides) -> argparse.Namespace:
79
"ssl_keyfile": None,
80
"ssl_certfile": None,
81
"ssl_ca_certs": None,
82
+ "ssl_cert_reqs": 0,
83
+ "ssl_ciphers": None,
84
"node_rank": 1,
85
"tensor_parallel_size": 1,
86
"pipeline_parallel_size": 1,
@@ -108,6 +114,8 @@ def _make_args(**overrides) -> argparse.Namespace:
114
ssl_keyfile=None,
115
ssl_certfile=None,
116
ssl_ca_certs=None,
117
+ ssl_cert_reqs=0,
118
+ ssl_ciphers=None,
119
node_rank=0,
120
tensor_parallel_size=1,
121
pipeline_parallel_size=1,
@@ -118,6 +126,33 @@ def _make_args(**overrides) -> argparse.Namespace:
126
return argparse.Namespace(**base)
127
128
129
+def _generate_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]:
130
+ """Generate a self-signed certificate for HTTPS lifecycle tests."""
131
+ cert_file = cert_dir / "cert.pem"
132
+ key_file = cert_dir / "key.pem"
133
+ subprocess.run(
134
+ [
135
+ "openssl",
136
+ "req",
137
+ "-x509",
138
+ "-newkey",
139
+ "rsa:2048",
140
+ "-keyout",
141
+ str(key_file),
142
+ "-out",
143
+ str(cert_file),
144
+ "-days",
145
+ "1",
146
+ "-nodes",
147
+ "-subj",
148
+ "/CN=localhost",
149
+ ],
150
+ check=True,
151
+ capture_output=True,
152
+ )
153
+ return cert_file, key_file
154
+
155
+
156
# ---------------------------------------------------------------------------
157
# Unit tests
158
# ---------------------------------------------------------------------------
@@ -141,6 +176,15 @@ def test_build_multi_port_external_lb_child_args_sets_external_rank_server():
176
assert child_args.api_server_count == 1
177
178
179
+def test_validate_multi_port_external_lb_args_allows_ssl():
180
+ args = _make_unit_args(
181
+ ssl_keyfile="/tmp/server.key",
182
+ ssl_certfile="/tmp/server.crt",
183
+ ssl_ca_certs="/tmp/ca.crt",
184
+ )
185
+ validate_multi_port_external_lb_args(args)
186
+
187
+
188
def test_aggregates_health():
189
supervisor = DPSupervisor(_make_unit_args())
190
supervisor._is_ready = True
@@ -236,10 +280,18 @@ class MockVLLMServer:
280
Health state is toggled by the test via set_healthy().
281
"""
282
239
- def __init__(self, port: int, drain_seconds: float = 0.0) -> None:
283
+ def __init__(
284
+ self,
285
+ port: int,
286
+ drain_seconds: float = 0.0,
287
+ ssl_keyfile: str | None = None,
288
+ ssl_certfile: str | None = None,
289
+ ) -> None:
290
self.port = port
291
self._healthy = False
292
self._drain_seconds = drain_seconds
293
+ self._ssl_keyfile = ssl_keyfile
294
+ self._ssl_certfile = ssl_certfile
295
self._server: uvicorn.Server | None = None
296
self._serve_task: asyncio.Task | None = None
297
@@ -274,6 +326,8 @@ class MockVLLMServer:
326
port=self.port,
327
log_level="warning",
328
lifespan="off",
329
+ ssl_keyfile=self._ssl_keyfile,
330
+ ssl_certfile=self._ssl_certfile,
331
)
332
self._server = uvicorn.Server(config)
333
@@ -312,7 +366,11 @@ class MockVLLMServer:
366
367
def launch_mock_vllm(child_args: argparse.Namespace, env_updates: dict[str, str]):
368
logger.info("Launching mock vLLM on port %s", child_args.port)
315
- mock_vllm = MockVLLMServer(port=child_args.port)
369
+ mock_vllm = MockVLLMServer(
370
+ port=child_args.port,
371
+ ssl_keyfile=child_args.ssl_keyfile,
372
+ ssl_certfile=child_args.ssl_certfile,
373
+ )
374
asyncio.run(mock_vllm.start())
375
376
@@ -320,7 +378,12 @@ def launch_mock_vllm_with_drain(
378
child_args: argparse.Namespace, env_updates: dict[str, str]
379
):
380
logger.info("Launching mock vLLM with 15s drain on port %s", child_args.port)
323
- mock_vllm = MockVLLMServer(port=child_args.port, drain_seconds=10.0)
381
+ mock_vllm = MockVLLMServer(
382
+ port=child_args.port,
383
+ drain_seconds=10.0,
384
+ ssl_keyfile=child_args.ssl_keyfile,
385
+ ssl_certfile=child_args.ssl_certfile,
386
+ )
387
asyncio.run(mock_vllm.start())
388
389
@@ -329,15 +392,16 @@ def launch_mock_vllm_with_drain(
392
# ---------------------------------------------------------------------------
393
394
332
-async def _poll_supervisor_health(expected_status: int) -> bool:
395
+async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -> bool:
396
"""
397
Poll GET /health on the supervisor until expected_status is seen.
398
A connection error is treated as 503-equivalent when expected_status != 200.
399
"""
337
- url = f"http://127.0.0.1:{_SUPERVISOR_PORT}/health"
400
+ scheme = "https" if use_ssl else "http"
401
+ url = f"{scheme}://127.0.0.1:{_SUPERVISOR_PORT}/health"
402
async with aiohttp.ClientSession() as session:
403
try:
340
- async with session.get(url) as resp:
404
+ async with session.get(url, ssl=False if use_ssl else None) as resp:
405
if resp.status != expected_status:
406
print(f"expected: {expected_status=}, got: {resp.status=}")
407
return False
@@ -349,12 +413,15 @@ async def _poll_supervisor_health(expected_status: int) -> bool:
413
return True
414
415
352
-async def _poll_until_api_server_running(port: int, retries: int = 10) -> None:
353
- url = f"http://127.0.0.1:{port}/health"
416
+async def _poll_until_api_server_running(
417
+ port: int, retries: int = 10, use_ssl: bool = False
418
+) -> None:
419
+ scheme = "https" if use_ssl else "http"
420
+ url = f"{scheme}://127.0.0.1:{port}/health"
421
async with aiohttp.ClientSession() as session:
422
for _ in range(retries):
423
try:
357
- async with session.get(url) as resp:
424
+ async with session.get(url, ssl=False if use_ssl else None) as resp:
425
if resp.status != 200:
426
return
427
await asyncio.sleep(1.0)
@@ -363,22 +430,34 @@ async def _poll_until_api_server_running(port: int, retries: int = 10) -> None:
430
await asyncio.sleep(1.0)
431
432
366
-async def _set_healthy(port: int) -> None:
367
- url = f"http://127.0.0.1:{port}/set_healthy"
368
- async with aiohttp.ClientSession() as session, session.get(url) as resp:
433
+async def _set_healthy(port: int, use_ssl: bool = False) -> None:
434
+ scheme = "https" if use_ssl else "http"
435
+ url = f"{scheme}://127.0.0.1:{port}/set_healthy"
436
+ async with (
437
+ aiohttp.ClientSession() as session,
438
+ session.get(url, ssl=False if use_ssl else None) as resp,
439
+ ):
440
assert resp.status == 200
441
442
372
-async def _set_unhealthy(port: int) -> None:
373
- url = f"http://127.0.0.1:{port}/set_unhealthy"
374
- async with aiohttp.ClientSession() as session, session.get(url) as resp:
443
+async def _set_unhealthy(port: int, use_ssl: bool = False) -> None:
444
+ scheme = "https" if use_ssl else "http"
445
+ url = f"{scheme}://127.0.0.1:{port}/set_unhealthy"
446
+ async with (
447
+ aiohttp.ClientSession() as session,
448
+ session.get(url, ssl=False if use_ssl else None) as resp,
449
+ ):
450
assert resp.status == 200
451
452
378
-async def _kill_server(port: int) -> None:
379
- url = f"http://127.0.0.1:{port}/kill"
453
+async def _kill_server(port: int, use_ssl: bool = False) -> None:
454
+ scheme = "https" if use_ssl else "http"
455
+ url = f"{scheme}://127.0.0.1:{port}/kill"
456
try:
381
- async with aiohttp.ClientSession() as session, session.get(url) as resp:
457
+ async with (
458
+ aiohttp.ClientSession() as session,
459
+ session.get(url, ssl=False if use_ssl else None) as resp,
460
+ ):
461
assert resp.status != 200
462
except Exception as e:
463
assert isinstance(e, aiohttp.ClientConnectorError)
@@ -455,6 +534,34 @@ async def test_basic_lifecycle(monkeypatch):
534
print("everything was cleaned up!")
535
536
537
+@pytest.mark.asyncio
538
+async def test_basic_lifecycle_with_ssl(monkeypatch):
539
+ with tempfile.TemporaryDirectory() as cert_dir:
540
+ cert_file, key_file = _generate_self_signed_cert(Path(cert_dir))
541
+ args = _make_args(
542
+ ssl_keyfile=str(key_file),
543
+ ssl_certfile=str(cert_file),
544
+ )
545
+
546
+ vllm_server_ports = [_CHILD_PORT_BASE + i for i in range(_N_CHILDREN)]
547
+
548
+ async with _run_supervisor(args, monkeypatch) as (supervisor, _task):
549
+ assert await _poll_supervisor_health(503, use_ssl=True)
550
+ assert not supervisor.is_ready
551
+
552
+ for port in vllm_server_ports:
553
+ assert await _poll_supervisor_health(503, use_ssl=True)
554
+ assert not supervisor.is_ready
555
+ await _poll_until_api_server_running(port, use_ssl=True)
556
+
557
+ for port in vllm_server_ports:
558
+ await _set_healthy(port, use_ssl=True)
559
+ await asyncio.sleep(1.0)
560
+
561
+ assert await _poll_supervisor_health(200, use_ssl=True)
562
+ assert supervisor.is_ready
563
+
564
+
565
@pytest.mark.asyncio
566
async def test_failed_startup(monkeypatch):
567
"""
vllm/entrypoints/openai/dp_supervisor.py
+18
-4
@@ -55,9 +55,9 @@ def validate_multi_port_external_lb_args(args: argparse.Namespace) -> None:
55
raise ValueError(
56
"Error: --data-parallel-multi-port-external-lb does not support --uds"
57
)
58
- if any((args.ssl_keyfile, args.ssl_certfile, args.ssl_ca_certs)):
58
+ if bool(args.ssl_keyfile) != bool(args.ssl_certfile):
59
raise ValueError(
60
- "Error: --data-parallel-multi-port-external-lb does not support HTTPS yet"
60
+ "Error: --ssl-keyfile and --ssl-certfile must be provided together"
61
)
62
if args.api_server_count not in (None, 1):
63
raise ValueError(
@@ -151,7 +151,8 @@ def _child_base_url(args: argparse.Namespace, port: int) -> str:
151
host = "127.0.0.1"
152
elif host == "::":
153
host = "::1"
154
- return f"http://{host}:{port}"
154
+ scheme = "https" if args.ssl_keyfile and args.ssl_certfile else "http"
155
+ return f"{scheme}://{host}:{port}"
156
157
158
def _join_processes_with_timeout(processes: list[BaseProcess], timeout: float) -> None:
@@ -178,7 +179,15 @@ async def _probe_endpoint(
179
"""
180
for iteration in range(conn_err_failure_threshold):
181
try:
181
- async with session.get(_child_base_url(args, port) + path) as response:
182
+ probe_ssl = None
183
+ if args.ssl_keyfile and args.ssl_certfile:
184
+ # Probes target node-local child servers over loopback, so skip
185
+ # certificate verification to avoid SAN/hostname mismatches for
186
+ # localhost/127.0.0.1 deployments.
187
+ probe_ssl = False
188
+ async with session.get(
189
+ _child_base_url(args, port) + path, ssl=probe_ssl
190
+ ) as response:
191
# vLLM returns 503 on EngineDeadError, so we should return
192
# immediately if vLLM responds with a non-200 status code.
193
return response.status == HTTPStatus.OK
@@ -272,6 +281,11 @@ class DPSupervisor:
281
host=host,
282
port=self.supervisor_port,
283
log_level=self.args.uvicorn_log_level,
284
+ ssl_keyfile=self.args.ssl_keyfile,
285
+ ssl_certfile=self.args.ssl_certfile,
286
+ ssl_ca_certs=self.args.ssl_ca_certs,
287
+ ssl_cert_reqs=self.args.ssl_cert_reqs,
288
+ ssl_ciphers=self.args.ssl_ciphers,
289
)
290
supervisor_server = uvicorn.Server(config)
291
supervisor_server_task = asyncio.create_task(