A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 14:13 UTC
add --insecure arg to the vllm bench to skip TLS (#34026)
Signed-off-by: Fan Yang <yan9fan@meta.com> Co-authored-by: Fan Yang <yan9fan@meta.com>
Fan Yang committed
Feb 10, 2026 at 06:23 UTC
a1946570d80c1bef78063e84b097951d8e8d4e6a
2 files changed
+139
-5
tests/benchmarks/test_serve_cli.py
+108
-1
@@ -1,15 +1,76 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import subprocess
4
+import tempfile
5
+import time
6
+from pathlib import Path
7
8
import pytest
9
+import requests
10
+import urllib3
11
12
from ..utils import RemoteOpenAIServer
13
14
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
15
16
12
-@pytest.fixture(scope="module")
17
+def generate_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]:
18
+ """Generate a self-signed certificate for testing."""
19
+ cert_file = cert_dir / "cert.pem"
20
+ key_file = cert_dir / "key.pem"
21
+
22
+ # Generate self-signed certificate using openssl
23
+ subprocess.run(
24
+ [
25
+ "openssl",
26
+ "req",
27
+ "-x509",
28
+ "-newkey",
29
+ "rsa:2048",
30
+ "-keyout",
31
+ str(key_file),
32
+ "-out",
33
+ str(cert_file),
34
+ "-days",
35
+ "1",
36
+ "-nodes",
37
+ "-subj",
38
+ "/CN=localhost",
39
+ ],
40
+ check=True,
41
+ capture_output=True,
42
+ )
43
+ return cert_file, key_file
44
+
45
+
46
+class RemoteOpenAIServerSSL(RemoteOpenAIServer):
47
+ """RemoteOpenAIServer subclass that supports SSL with self-signed certs."""
48
+
49
+ @property
50
+ def url_root(self) -> str:
51
+ return f"https://{self.host}:{self.port}"
52
+
53
+ def _wait_for_server(self, *, url: str, timeout: float):
54
+ """Override to use HTTPS with SSL verification disabled."""
55
+ # Suppress InsecureRequestWarning for self-signed certs
56
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
57
+
58
+ start = time.time()
59
+ while True:
60
+ try:
61
+ if requests.get(url, verify=False).status_code == 200:
62
+ break
63
+ except Exception:
64
+ result = self._poll()
65
+ if result is not None and result != 0:
66
+ raise RuntimeError("Server exited unexpectedly.") from None
67
+
68
+ time.sleep(0.5)
69
+ if time.time() - start > timeout:
70
+ raise RuntimeError("Server failed to start in time.") from None
71
+
72
+
73
+@pytest.fixture(scope="function")
74
def server():
75
args = ["--max-model-len", "1024", "--enforce-eager", "--load-format", "dummy"]
76
@@ -17,6 +78,27 @@ def server():
78
yield remote_server
79
80
81
+@pytest.fixture(scope="function")
82
+def ssl_server():
83
+ """Start a vLLM server with SSL enabled using a self-signed certificate."""
84
+ with tempfile.TemporaryDirectory() as cert_dir:
85
+ cert_file, key_file = generate_self_signed_cert(Path(cert_dir))
86
+ args = [
87
+ "--max-model-len",
88
+ "1024",
89
+ "--enforce-eager",
90
+ "--load-format",
91
+ "dummy",
92
+ "--ssl-certfile",
93
+ str(cert_file),
94
+ "--ssl-keyfile",
95
+ str(key_file),
96
+ ]
97
+
98
+ with RemoteOpenAIServerSSL(MODEL_NAME, args) as remote_server:
99
+ yield remote_server
100
+
101
+
102
@pytest.mark.benchmark
103
def test_bench_serve(server):
104
# Test default model detection and input/output len
@@ -42,6 +124,31 @@ def test_bench_serve(server):
124
assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
125
126
127
+@pytest.mark.benchmark
128
+def test_bench_serve_insecure(ssl_server):
129
+ """Test --insecure flag with an HTTPS server using a self-signed certificate."""
130
+ base_url = f"https://{ssl_server.host}:{ssl_server.port}"
131
+ command = [
132
+ "vllm",
133
+ "bench",
134
+ "serve",
135
+ "--base-url",
136
+ base_url,
137
+ "--input-len",
138
+ "32",
139
+ "--output-len",
140
+ "4",
141
+ "--num-prompts",
142
+ "5",
143
+ "--insecure",
144
+ ]
145
+ result = subprocess.run(command, capture_output=True, text=True)
146
+ print(result.stdout)
147
+ print(result.stderr)
148
+
149
+ assert result.returncode == 0, f"Benchmark failed: {result.stderr}"
150
+
151
+
152
@pytest.mark.benchmark
153
def test_bench_serve_chat(server):
154
command = [
vllm/benchmarks/serve.py
+31
-4
@@ -26,6 +26,7 @@ import json
26
import os
27
import random
28
import shutil
29
+import ssl
30
import time
31
import uuid
32
import warnings
@@ -60,11 +61,14 @@ TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) a
61
62
63
async def get_first_model_from_server(
63
- base_url: str, headers: dict | None = None
64
+ base_url: str,
65
+ headers: dict | None = None,
66
+ ssl_context: ssl.SSLContext | bool | None = None,
67
) -> tuple[str, str]:
68
"""Fetch the first model from the server's /v1/models endpoint."""
69
models_url = f"{base_url}/v1/models"
67
- async with aiohttp.ClientSession() as session:
70
+ connector = aiohttp.TCPConnector(ssl=ssl_context)
71
+ async with aiohttp.ClientSession(connector=connector) as session:
72
try:
73
async with session.get(models_url, headers=headers) as response:
74
response.raise_for_status()
@@ -619,6 +623,7 @@ async def benchmark(
623
ramp_up_start_rps: int | None = None,
624
ramp_up_end_rps: int | None = None,
625
ready_check_timeout_sec: int = 600,
626
+ ssl_context: ssl.SSLContext | bool | None = None,
627
):
628
try:
629
request_func = ASYNC_REQUEST_FUNCS[endpoint_type]
@@ -626,6 +631,8 @@ async def benchmark(
631
raise ValueError(f"Unknown backend: {endpoint_type}") from None
632
633
# Reuses connections across requests to reduce TLS handshake overhead.
634
+ # Use ssl_context if provided, otherwise default to True for https URLs
635
+ ssl_setting = ssl_context if ssl_context is not None else ("https://" in api_url)
636
connector = aiohttp.TCPConnector(
637
limit=max_concurrency or 0,
638
limit_per_host=max_concurrency or 0,
@@ -634,7 +641,7 @@ async def benchmark(
641
keepalive_timeout=60,
642
enable_cleanup_closed=True,
643
force_close=False,
637
- ssl=("https://" in api_url),
644
+ ssl=ssl_setting,
645
)
646
647
session = aiohttp.ClientSession(
@@ -1513,6 +1520,14 @@ def add_cli_args(parser: argparse.ArgumentParser):
1520
default=None,
1521
)
1522
1523
+ parser.add_argument(
1524
+ "--insecure",
1525
+ action="store_true",
1526
+ default=False,
1527
+ help="Disable SSL certificate verification. Use this option when "
1528
+ "connecting to servers with self-signed certificates.",
1529
+ )
1530
+
1531
1532
def main(args: argparse.Namespace) -> dict[str, Any]:
1533
return asyncio.run(main_async(args))
@@ -1564,10 +1579,21 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1579
else:
1580
raise ValueError("Invalid header format. Please use KEY=VALUE format.")
1581
1582
+ # SSL context configuration
1583
+ ssl_context: ssl.SSLContext | bool | None = None
1584
+ if args.insecure:
1585
+ # Disable SSL certificate verification
1586
+ ssl_context = False
1587
+ elif "https://" in base_url:
1588
+ # Use default SSL context for HTTPS
1589
+ ssl_context = True
1590
+
1591
# Fetch model from server if not specified
1592
if args.model is None:
1593
print("Model not specified, fetching first model from server...")
1570
- model_name, model_id = await get_first_model_from_server(base_url, headers)
1594
+ model_name, model_id = await get_first_model_from_server(
1595
+ base_url, headers, ssl_context
1596
+ )
1597
print(f"First model name: {model_name}, first model id: {model_id}")
1598
else:
1599
model_name = args.served_model_name
@@ -1691,6 +1717,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1717
ramp_up_start_rps=args.ramp_up_start_rps,
1718
ramp_up_end_rps=args.ramp_up_end_rps,
1719
ready_check_timeout_sec=args.ready_check_timeout_sec,
1720
+ ssl_context=ssl_context,
1721
)
1722
1723
# Save config and results to json