@tduncan / Nvidia-Optimization-Stack-VLLM / commits / b8cb75b149

[Rust Frontend] Add static HTTPS and mTLS support for HTTP and gRPC (#45890)

Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Tahsin Tunan committed Jun 30, 2026 at 07:45 UTC b8cb75b1497a9bdae39964b3beee34eaaf5a678f
16 files changed +1942 -125
rust/Cargo.lock
+69 -20
@@ -272,6 +272,18 @@ version = "1.1.2"
272 source = "registry+https://github.com/rust-lang/crates.io-index"
273 checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
274
275 +[[package]]
276 +name = "auto_enums"
277 +version = "0.8.9"
278 +source = "registry+https://github.com/rust-lang/crates.io-index"
279 +checksum = "2e4487600931c9a89f8db7ffbdf3fbdd45bb7bd85e26861f659a463cd0dff966"
280 +dependencies = [
281 + "derive_utils",
282 + "proc-macro2",
283 + "quote",
284 + "syn 2.0.117",
285 +]
286 +
287 [[package]]
288 name = "auto_impl"
289 version = "1.3.0"
@@ -938,6 +950,17 @@ dependencies = [
950 "unicode-xid",
951 ]
952
953 +[[package]]
954 +name = "derive_utils"
955 +version = "0.15.1"
956 +source = "registry+https://github.com/rust-lang/crates.io-index"
957 +checksum = "362f47930db19fe7735f527e6595e4900316b893ebf6d48ad3d31be928d57dd6"
958 +dependencies = [
959 + "proc-macro2",
960 + "quote",
961 + "syn 2.0.117",
962 +]
963 +
964 [[package]]
965 name = "digest"
966 version = "0.10.7"
@@ -1478,9 +1501,9 @@ dependencies = [
1501
1502 [[package]]
1503 name = "h2"
1481 -version = "0.4.13"
1504 +version = "0.4.15"
1505 source = "registry+https://github.com/rust-lang/crates.io-index"
1483 -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
1506 +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
1507 dependencies = [
1508 "atomic-waker",
1509 "bytes",
@@ -1638,9 +1661,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
1661
1662 [[package]]
1663 name = "hyper"
1641 -version = "1.8.1"
1664 +version = "1.10.1"
1665 source = "registry+https://github.com/rust-lang/crates.io-index"
1643 -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
1666 +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
1667 dependencies = [
1668 "atomic-waker",
1669 "bytes",
@@ -1653,7 +1676,6 @@ dependencies = [
1676 "httpdate",
1677 "itoa",
1678 "pin-project-lite",
1656 - "pin-utils",
1679 "smallvec",
1680 "tokio",
1681 "want",
@@ -2569,15 +2591,14 @@ dependencies = [
2591
2592 [[package]]
2593 name = "openssl"
2572 -version = "0.10.76"
2594 +version = "0.10.81"
2595 source = "registry+https://github.com/rust-lang/crates.io-index"
2574 -checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
2596 +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
2597 dependencies = [
2598 "bitflags",
2599 "cfg-if",
2600 "foreign-types",
2601 "libc",
2580 - "once_cell",
2602 "openssl-macros",
2603 "openssl-sys",
2604 ]
@@ -2610,9 +2631,9 @@ dependencies = [
2631
2632 [[package]]
2633 name = "openssl-sys"
2613 -version = "0.9.112"
2634 +version = "0.9.117"
2635 source = "registry+https://github.com/rust-lang/crates.io-index"
2615 -checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
2636 +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
2637 dependencies = [
2638 "cc",
2639 "libc",
@@ -2783,12 +2804,6 @@ version = "0.2.17"
2804 source = "registry+https://github.com/rust-lang/crates.io-index"
2805 checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
2806
2786 -[[package]]
2787 -name = "pin-utils"
2788 -version = "0.1.0"
2789 -source = "registry+https://github.com/rust-lang/crates.io-index"
2790 -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
2791 -
2807 [[package]]
2808 name = "pkg-config"
2809 version = "0.3.32"
@@ -2988,7 +3003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3003 checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
3004 dependencies = [
3005 "heck",
2991 - "itertools 0.10.5",
3006 + "itertools 0.14.0",
3007 "log",
3008 "multimap",
3009 "petgraph",
@@ -3009,7 +3024,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3024 checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
3025 dependencies = [
3026 "anyhow",
3012 - "itertools 0.10.5",
3027 + "itertools 0.14.0",
3028 "proc-macro2",
3029 "quote",
3030 "syn 2.0.117",
@@ -3503,9 +3518,9 @@ dependencies = [
3518
3519 [[package]]
3520 name = "rustls-pki-types"
3506 -version = "1.14.0"
3521 +version = "1.14.1"
3522 source = "registry+https://github.com/rust-lang/crates.io-index"
3508 -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
3523 +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
3524 dependencies = [
3525 "zeroize",
3526 ]
@@ -4385,6 +4400,22 @@ dependencies = [
4400 "serde_json",
4401 ]
4402
4403 +[[package]]
4404 +name = "tls-listener"
4405 +version = "0.11.2"
4406 +source = "registry+https://github.com/rust-lang/crates.io-index"
4407 +checksum = "1461056cc1ef47003f7ee16e4cef3741068d4c7f6b627bfce49b7c00c120a530"
4408 +dependencies = [
4409 + "axum",
4410 + "futures-util",
4411 + "openssl",
4412 + "pin-project-lite",
4413 + "thiserror 2.0.18",
4414 + "tokio",
4415 + "tokio-openssl",
4416 + "tracing",
4417 +]
4418 +
4419 [[package]]
4420 name = "tokenizers"
4421 version = "0.22.2"
@@ -4457,6 +4488,17 @@ dependencies = [
4488 "tokio",
4489 ]
4490
4491 +[[package]]
4492 +name = "tokio-openssl"
4493 +version = "0.6.5"
4494 +source = "registry+https://github.com/rust-lang/crates.io-index"
4495 +checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd"
4496 +dependencies = [
4497 + "openssl",
4498 + "openssl-sys",
4499 + "tokio",
4500 +]
4501 +
4502 [[package]]
4503 name = "tokio-rustls"
4504 version = "0.26.4"
@@ -5220,6 +5262,7 @@ dependencies = [
5262 "anyhow",
5263 "async-openai",
5264 "asynk-strim-attr",
5265 + "auto_enums",
5266 "axum",
5267 "bytes",
5268 "clap",
@@ -5227,10 +5270,13 @@ dependencies = [
5270 "expect-test",
5271 "futures",
5272 "http-body",
5273 + "hyper",
5274 + "hyper-util",
5275 "indexmap 2.13.0",
5276 "itertools 0.14.0",
5277 "libc",
5278 "llm-multimodal",
5279 + "openssl",
5280 "prost",
5281 "prost-types",
5282 "rmp-serde",
@@ -5242,8 +5288,11 @@ dependencies = [
5288 "sha2",
5289 "socket2",
5290 "subtle",
5291 + "tempfile",
5292 "thiserror-ext",
5293 + "tls-listener",
5294 "tokio",
5295 + "tokio-openssl",
5296 "tokio-stream",
5297 "tokio-util",
5298 "tonic",
rust/Cargo.toml
+10
@@ -26,6 +26,7 @@ arc-swap = "1.9.0"
26 async-openai = { version = "0.33.1", default-features = false, features = ["native-tls"] }
27 async-trait = "0.1.89"
28 asynk-strim-attr = "0.1.0"
29 +auto_enums = { version = "0.8.9", features = ["tokio1"] }
30 axum = "0.8.8"
31 base64 = "0.22.1"
32 bytemuck = { version = "1.25.0", features = ["extern_crate_alloc"] }
@@ -43,6 +44,12 @@ half = { version = "2.7.1", features = ["bytemuck"] }
44 hex = "0.4.3"
45 hf-hub = { version = "0.5.0", default-features = false, features = ["tokio"] }
46 http-body = "1.0.1"
47 +hyper = { version = "1.10.1", features = ["http1", "server"] }
48 +hyper-util = { version = "0.1.20", features = [
49 + "server-graceful",
50 + "service",
51 + "tokio",
52 +] }
53 indexmap = "2.13.0"
54 itertools = "0.14.0"
55 libc = "0.2.177"
@@ -54,6 +61,7 @@ native-tls-vendored = { package = "native-tls", version = "0.2.18", features = [
61 ndarray = { version = "0.16.1", features = ["serde"] }
62 openai-harmony = { package = "oss-harmony", git = "https://github.com/oss-harmony/harmony", tag = "v0.0.11", default-features = false }
63 openai-protocol = "1.6.0"
64 +openssl = "0.10"
65 parking_lot = "0.12.5"
66 paste = "1.0.15"
67 prometheus-client = "0.24.0"
@@ -89,6 +97,7 @@ thiserror = "2.0.16"
97 thiserror-ext = "0.3.0"
98 tiktoken-rs = "0.9.1"
99 time = { version = "0.3.47", features = ["formatting", "local-offset", "macros"] }
100 +tls-listener = { version = "0.11.2", default-features = false, features = ["openssl", "tokio-net", "axum"] }
101 tokenizers = "0.22.0"
102 tokio = { version = "1.47.1", features = [
103 "macros",
@@ -97,6 +106,7 @@ tokio = { version = "1.47.1", features = [
106 "sync",
107 "time",
108 ] }
109 +tokio-openssl = "0.6"
110 tokio-stream = "0.1"
111 tokio-util = { version = "0.7.18", features = ["rt"] }
112 tonic = "0.14.5"
rust/src/cmd/src/cli.rs
+66 -1
@@ -25,7 +25,7 @@ use vllm_managed_engine::ManagedEngineConfig;
25 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
26 use vllm_server::{
27 ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
28 - HttpListenerMode, ParserSelection, RendererSelection,
28 + DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
29 };
30
31 use crate::cli::unsupported::UnsupportedArgs;
@@ -154,6 +154,11 @@ pub struct SharedRuntimeArgs {
154 #[arg(long, default_value_t = 0)]
155 #[serde(default)]
156 pub shutdown_timeout: u64,
157 + /// Maximum idle time (seconds) on a keep-alive HTTP connection before the
158 + /// server closes it (default 5).
159 + #[arg(long = "http-timeout-keep-alive", env = "VLLM_HTTP_TIMEOUT_KEEP_ALIVE")]
160 + #[serde(default)]
161 + pub http_timeout_keep_alive: Option<u64>,
162
163 /// The file path to the chat template, or the template in single-line form
164 /// for the specified model.
@@ -257,6 +262,34 @@ pub struct SharedRuntimeArgs {
262 #[serde(default)]
263 pub allow_credentials: bool,
264
265 + /// The file path to the SSL key file. When omitted, the key is read from
266 + /// `--ssl-certfile` (combined PEM).
267 + #[arg(long)]
268 + #[serde(default)]
269 + pub ssl_keyfile: Option<String>,
270 +
271 + /// The file path to the SSL cert file. Enables TLS when set.
272 + #[arg(long)]
273 + #[serde(default)]
274 + pub ssl_certfile: Option<String>,
275 +
276 + /// The CA certificates file used to verify client certificates (mTLS).
277 + #[arg(long)]
278 + #[serde(default)]
279 + pub ssl_ca_certs: Option<String>,
280 +
281 + /// Whether a client certificate is required: 0 = none, 1 = optional,
282 + /// 2 = required (mirrors Python's `ssl.CERT_*`).
283 + #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
284 + #[serde(default)]
285 + pub ssl_cert_reqs: i32,
286 +
287 + /// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
288 + /// When unset, the linked OpenSSL's default suites are used.
289 + #[arg(long)]
290 + #[serde(default)]
291 + pub ssl_ciphers: Option<String>,
292 +
293 /// Unsupported Python vLLM frontend arguments recognized but not yet
294 /// implemented in Rust.
295 #[educe(Debug(ignore))]
@@ -277,6 +310,13 @@ impl SharedRuntimeArgs {
310 Duration::from_secs(self.shutdown_timeout)
311 }
312
313 + /// Maximum idle time on a keep-alive HTTP connection before the server
314 + /// closes it.
315 + pub fn keep_alive_timeout(&self) -> Duration {
316 + self.http_timeout_keep_alive
317 + .map_or(DEFAULT_KEEP_ALIVE_TIMEOUT, Duration::from_secs)
318 + }
319 +
320 /// Apply fallback logic for API key configuration from env variables.
321 fn apply_env_api_key_fallback(&mut self) {
322 if self.api_key.is_empty()
@@ -301,8 +341,10 @@ impl SharedRuntimeArgs {
341 ) -> Config {
342 let ready_timeout = self.ready_timeout();
343 let shutdown_timeout = self.shutdown_timeout();
344 + let keep_alive_timeout = self.keep_alive_timeout();
345 let api_server_options = self.api_server_options();
346 let cors = self.cors_config();
347 + let tls = self.tls_config();
348
349 Config {
350 transport_mode: TransportMode::Bootstrapped {
@@ -329,10 +371,12 @@ impl SharedRuntimeArgs {
371 max_logprobs: self.max_logprobs,
372 api_server_options,
373 cors,
374 + tls,
375 api_keys: self.api_key,
376 disable_log_stats: self.disable_log_stats,
377 grpc_port: self.grpc_port,
378 shutdown_timeout,
379 + keep_alive_timeout,
380 }
381 }
382
@@ -349,8 +393,10 @@ impl SharedRuntimeArgs {
393 ) -> Config {
394 let ready_timeout = self.ready_timeout();
395 let shutdown_timeout = self.shutdown_timeout();
396 + let keep_alive_timeout = self.keep_alive_timeout();
397 let api_server_options = self.api_server_options();
398 let cors = self.cors_config();
399 + let tls = self.tls_config();
400
401 Config {
402 transport_mode: TransportMode::HandshakeOwner {
@@ -375,10 +421,12 @@ impl SharedRuntimeArgs {
421 max_logprobs: self.max_logprobs,
422 api_server_options,
423 cors,
424 + tls,
425 api_keys: self.api_key,
426 disable_log_stats: self.disable_log_stats,
427 grpc_port: self.grpc_port,
428 shutdown_timeout,
429 + keep_alive_timeout,
430 }
431 }
432
@@ -398,6 +446,23 @@ impl SharedRuntimeArgs {
446 allow_credentials: self.allow_credentials,
447 }
448 }
449 +
450 + /// Build the TLS config: `Some` when any `ssl_*` argument is set, else
451 + /// `None` (plaintext). The combination is validated in [`Config::validate`].
452 + fn tls_config(&self) -> Option<TlsConfig> {
453 + let tls_requested = self.ssl_certfile.is_some()
454 + || self.ssl_keyfile.is_some()
455 + || self.ssl_ca_certs.is_some()
456 + || self.ssl_cert_reqs != 0
457 + || self.ssl_ciphers.is_some();
458 + tls_requested.then(|| TlsConfig {
459 + cert_file: self.ssl_certfile.clone(),
460 + key_file: self.ssl_keyfile.clone(),
461 + ca_certs: self.ssl_ca_certs.clone(),
462 + cert_reqs: self.ssl_cert_reqs,
463 + ciphers: self.ssl_ciphers.clone(),
464 + })
465 + }
466 }
467
468 fn default_engine_ready_timeout_secs() -> u64 {
rust/src/cmd/src/cli/tests.rs
+167 -9
@@ -41,6 +41,7 @@ fn serve_args_forward_python_flags_with_separator() {
41 max_logprobs: None,
42 grpc_port: None,
43 shutdown_timeout: 0,
44 + http_timeout_keep_alive: None,
45 chat_template: None,
46 default_chat_template_kwargs: None,
47 chat_template_content_format: Auto,
@@ -65,6 +66,11 @@ fn serve_args_forward_python_flags_with_separator() {
66 ],
67 ),
68 allow_credentials: false,
69 + ssl_keyfile: None,
70 + ssl_certfile: None,
71 + ssl_ca_certs: None,
72 + ssl_cert_reqs: 0,
73 + ssl_ciphers: None,
74 },
75 managed_engine: ManagedEngineArgs {
76 python: "../vllm/.venv/bin/python",
@@ -363,6 +369,140 @@ fn serve_passes_enable_prompt_tokens_details_into_config() {
369 assert!(config.api_server_options.enable_prompt_tokens_details);
370 }
371
372 +#[test]
373 +fn serve_passes_tls_into_config() {
374 + let cli = Cli::try_parse_from([
375 + "vllm-rs",
376 + "serve",
377 + "Qwen/Qwen3-0.6B",
378 + "--ssl-certfile",
379 + "/tmp/cert.pem",
380 + "--ssl-keyfile",
381 + "/tmp/key.pem",
382 + "--ssl-ca-certs",
383 + "/tmp/ca.pem",
384 + "--ssl-cert-reqs",
385 + "2",
386 + ])
387 + .unwrap();
388 +
389 + let Command::Serve(args) = cli.command else {
390 + panic!("expected serve args");
391 + };
392 + let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
393 + let tls = config.tls.expect("tls configured");
394 + assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
395 + assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
396 + assert_eq!(tls.ca_certs.as_deref(), Some("/tmp/ca.pem"));
397 + assert_eq!(tls.cert_reqs, 2);
398 +}
399 +
400 +#[test]
401 +fn serve_without_ssl_flags_has_no_tls() {
402 + let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
403 +
404 + let Command::Serve(args) = cli.command else {
405 + panic!("expected serve args");
406 + };
407 + let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
408 + assert!(config.tls.is_none());
409 +}
410 +
411 +#[test]
412 +fn serve_ssl_keyfile_without_certfile_fails_validation() {
413 + let cli = Cli::try_parse_from([
414 + "vllm-rs",
415 + "serve",
416 + "Qwen/Qwen3-0.6B",
417 + "--ssl-keyfile",
418 + "/tmp/key.pem",
419 + ])
420 + .unwrap();
421 +
422 + let Command::Serve(args) = cli.command else {
423 + panic!("expected serve args");
424 + };
425 + let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
426 + // TLS is requested (a key was given) but there is no certificate, so
427 + // validation fails loud rather than silently serving plaintext.
428 + assert_eq!(config.tls.as_ref().expect("tls requested").cert_file, None);
429 + let err = config.validate().unwrap_err().to_string();
430 + assert!(err.contains("--ssl-certfile is required"), "{err}");
431 +}
432 +
433 +#[test]
434 +fn serve_mtls_without_ca_certs_fails_validation() {
435 + let cli = Cli::try_parse_from([
436 + "vllm-rs",
437 + "serve",
438 + "Qwen/Qwen3-0.6B",
439 + "--ssl-certfile",
440 + "/tmp/cert.pem",
441 + "--ssl-cert-reqs",
442 + "2",
443 + ])
444 + .unwrap();
445 +
446 + let Command::Serve(args) = cli.command else {
447 + panic!("expected serve args");
448 + };
449 + let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
450 + // Client-cert verification without a CA bundle has nothing to verify
451 + // against, so it fails loud at startup.
452 + let err = config.validate().unwrap_err().to_string();
453 + assert!(err.contains("--ssl-ca-certs is required"), "{err}");
454 +}
455 +
456 +#[test]
457 +fn frontend_args_json_passes_tls_into_config() {
458 + let cli = Cli::try_parse_from([
459 + "vllm-rs",
460 + "frontend",
461 + "--listen-fd",
462 + "3",
463 + "--input-address",
464 + "ipc:///tmp/input.sock",
465 + "--output-address",
466 + "ipc:///tmp/output.sock",
467 + "--args-json",
468 + r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_keyfile":"/tmp/key.pem"}"#,
469 + ])
470 + .unwrap();
471 +
472 + let Command::Frontend(args) = cli.command else {
473 + panic!("expected frontend args");
474 + };
475 + let config = args.into_config();
476 + let tls = config.tls.expect("tls configured");
477 + assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
478 + assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
479 +}
480 +
481 +#[test]
482 +fn frontend_args_json_rejects_out_of_range_cert_reqs() {
483 + let cli = Cli::try_parse_from([
484 + "vllm-rs",
485 + "frontend",
486 + "--listen-fd",
487 + "3",
488 + "--input-address",
489 + "ipc:///tmp/input.sock",
490 + "--output-address",
491 + "ipc:///tmp/output.sock",
492 + "--args-json",
493 + r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_cert_reqs":5}"#,
494 + ])
495 + .unwrap();
496 +
497 + let Command::Frontend(args) = cli.command else {
498 + panic!("expected frontend args");
499 + };
500 + // The JSON path bypasses clap's range check, so validate() is the only guard.
501 + let config = args.into_config();
502 + let err = config.validate().unwrap_err().to_string();
503 + assert!(err.contains("--ssl-cert-reqs"), "{err}");
504 +}
505 +
506 #[test]
507 fn frontend_args_json_passes_enable_request_id_headers_into_config() {
508 let cli = Cli::try_parse_from([
@@ -481,13 +621,13 @@ fn serve_args_reject_unsupported_flag_arg() {
621 "vllm-rs",
622 "serve",
623 "Qwen/Qwen3-0.6B",
484 - "--ssl-keyfile",
485 - "/tmp/key.pem",
624 + "--root-path",
625 + "/prefix",
626 ])
627 .unwrap_err();
628
629 expect![[r#"
490 - error: invalid value '/tmp/key.pem' for '--ssl-keyfile <SSL_KEYFILE>': argument is not implemented in Rust frontend yet
630 + error: invalid value '/prefix' for '--root-path <ROOT_PATH>': argument is not implemented in Rust frontend yet
631
632 Remove this unsupported argument to continue.
633
@@ -562,6 +702,7 @@ fn frontend_args_accept_json() {
702 max_logprobs: None,
703 grpc_port: None,
704 shutdown_timeout: 0,
705 + http_timeout_keep_alive: None,
706 chat_template: None,
707 default_chat_template_kwargs: None,
708 chat_template_content_format: Auto,
@@ -586,6 +727,11 @@ fn frontend_args_accept_json() {
727 ],
728 ),
729 allow_credentials: false,
730 + ssl_keyfile: None,
731 + ssl_certfile: None,
732 + ssl_ca_certs: None,
733 + ssl_cert_reqs: 0,
734 + ssl_ciphers: None,
735 },
736 },
737 ),
@@ -798,14 +944,14 @@ fn frontend_args_json_rejects_unsupported_fields() {
944 "--output-address",
945 "ipc:///tmp/output.sock",
946 "--args-json",
801 - r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}"#,
947 + r#"{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}"#,
948 ])
949 .unwrap_err();
950
951 expect![[r#"
806 - error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
952 + error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}' for '--args-json <JSON>':
953 The following arguments are not implemented in Rust frontend yet:
808 - - ssl_keyfile
954 + - root_path
955
956 Remove these arguments to continue.
957
@@ -825,16 +971,16 @@ fn frontend_args_json_aggregates_multiple_unsupported_fields() {
971 "--output-address",
972 "ipc:///tmp/output.sock",
973 "--args-json",
828 - r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}"#,
974 + r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}"#,
975 ])
976 .unwrap_err();
977
978 let actual = error.to_string().replace(": \n", ":\n");
979 expect![[r#"
834 - error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
980 + error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}' for '--args-json <JSON>':
981 The following arguments are not implemented in Rust frontend yet:
982 - response_role
837 - - ssl_keyfile
983 + - root_path
984
985 Remove these arguments to continue.
986
@@ -1077,6 +1223,7 @@ fn serve_args_accept_handshake_aliases() {
1223 max_logprobs: None,
1224 grpc_port: None,
1225 shutdown_timeout: 0,
1226 + http_timeout_keep_alive: None,
1227 chat_template: None,
1228 default_chat_template_kwargs: None,
1229 chat_template_content_format: Auto,
@@ -1101,6 +1248,11 @@ fn serve_args_accept_handshake_aliases() {
1248 ],
1249 ),
1250 allow_credentials: false,
1251 + ssl_keyfile: None,
1252 + ssl_certfile: None,
1253 + ssl_ca_certs: None,
1254 + ssl_cert_reqs: 0,
1255 + ssl_ciphers: None,
1256 },
1257 managed_engine: ManagedEngineArgs {
1258 python: "python3",
@@ -1234,10 +1386,12 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1386 ],
1387 allow_credentials: false,
1388 },
1389 + tls: None,
1390 api_keys: [],
1391 disable_log_stats: false,
1392 grpc_port: None,
1393 shutdown_timeout: 0ns,
1394 + keep_alive_timeout: 5s,
1395 }
1396 "#]]
1397 .assert_debug_eq(&Config {
@@ -1315,10 +1469,12 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1469 ],
1470 allow_credentials: false,
1471 },
1472 + tls: None,
1473 api_keys: [],
1474 disable_log_stats: false,
1475 grpc_port: None,
1476 shutdown_timeout: 0ns,
1477 + keep_alive_timeout: 5s,
1478 }
1479 "#]]
1480 .assert_debug_eq(&config);
@@ -1414,10 +1570,12 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
1570 ],
1571 allow_credentials: false,
1572 },
1573 + tls: None,
1574 api_keys: [],
1575 disable_log_stats: false,
1576 grpc_port: None,
1577 shutdown_timeout: 0ns,
1578 + keep_alive_timeout: 5s,
1579 }
1580 "#]]
1581 .assert_debug_eq(&config);
rust/src/cmd/src/cli/unsupported.rs
-21
@@ -526,18 +526,6 @@ pub struct ServerUnsupportedArgs {
526 #[arg(long)]
527 pub disable_access_log_for_endpoints: Option<Noop>,
528
529 - /// The file path to the SSL key file.
530 - #[arg(long)]
531 - pub ssl_keyfile: Option<Unsupported>,
532 -
533 - /// The file path to the SSL cert file.
534 - #[arg(long)]
535 - pub ssl_certfile: Option<Unsupported>,
536 -
537 - /// The CA certificates file.
538 - #[arg(long)]
539 - pub ssl_ca_certs: Option<Unsupported>,
540 -
529 /// Refresh SSL Context when SSL certificate files change
530 #[arg(
531 long,
@@ -547,15 +535,6 @@ pub struct ServerUnsupportedArgs {
535 )]
536 pub enable_ssl_refresh: Option<Unsupported>,
537
550 - /// Whether client certificate is required (see stdlib ssl module's).
551 - #[arg(long)]
552 - pub ssl_cert_reqs: Option<Unsupported>,
553 -
554 - /// SSL cipher suites for HTTPS (TLS 1.2 and below only).
555 - /// Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'
556 - #[arg(long)]
557 - pub ssl_ciphers: Option<Unsupported>,
558 -
538 /// FastAPI root_path when app is behind a path based routing proxy.
539 #[arg(long)]
540 pub root_path: Option<Unsupported>,
rust/src/engine-core-client/src/tests/client.rs
+13 -7
@@ -1285,18 +1285,24 @@ async fn dropping_multiple_live_streams_aborts_all_in_a_burst() {
1285 )
1286 .await;
1287
1288 - let abort =
1289 - timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
1290 - assert_eq!(abort[0].as_ref(), &[0x01]);
1291 - let ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
1288 + // Aborts may coalesce into one burst or split across several.
1289 + let mut aborted = BTreeSet::new();
1290 + while aborted.len() < 3 {
1291 + let abort =
1292 + timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
1293 + assert_eq!(abort[0].as_ref(), &[0x01]);
1294 + let ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
1295 + aborted.extend(ids);
1296 + }
1297 assert_eq!(
1293 - ids,
1294 - vec![
1298 + aborted,
1299 + BTreeSet::from([
1300 "req-1".to_string(),
1301 "req-2".to_string(),
1302 "req-3".to_string()
1298 - ]
1303 + ])
1304 );
1305 + // No spurious extra aborts.
1306 assert!(
1307 timeout(Duration::from_millis(100), recv_engine_message(dealer)).await.is_err()
1308 );
rust/src/server/Cargo.toml
+7
@@ -7,14 +7,18 @@ license.workspace = true
7 [dependencies]
8 anyhow.workspace = true
9 asynk-strim-attr.workspace = true
10 +auto_enums.workspace = true
11 axum.workspace = true
12 educe.workspace = true
13 futures.workspace = true
14 http-body.workspace = true
15 +hyper.workspace = true
16 +hyper-util.workspace = true
17 indexmap.workspace = true
18 itertools.workspace = true
19 libc.workspace = true
20 llm-multimodal.workspace = true
21 +openssl.workspace = true
22 prost.workspace = true
23 prost-types.workspace = true
24 rmpv.workspace = true
@@ -25,7 +29,9 @@ sha2.workspace = true
29 socket2.workspace = true
30 subtle.workspace = true
31 thiserror-ext.workspace = true
32 +tls-listener.workspace = true
33 tokio.workspace = true
34 +tokio-openssl.workspace = true
35 tokio-stream.workspace = true
36 tokio-util.workspace = true
37 tonic.workspace = true
@@ -54,6 +60,7 @@ clap.workspace = true
60 expect-test.workspace = true
61 rmp-serde.workspace = true
62 serial_test.workspace = true
63 +tempfile.workspace = true
64 tower.workspace = true
65 vllm-engine-core-client = { workspace = true, features = ["test-util"] }
66 zeromq.workspace = true
rust/src/server/examples/external_engine_openai_qwen.rs
+2
@@ -71,10 +71,12 @@ async fn main() -> Result<()> {
71 max_logprobs: None,
72 api_server_options: ApiServerOptions::default(),
73 cors: CorsConfig::default(),
74 + tls: None,
75 api_keys: Vec::new(),
76 disable_log_stats: false,
77 grpc_port: None,
78 shutdown_timeout: Duration::ZERO,
79 + keep_alive_timeout: Duration::from_secs(5),
80 };
81
82 let bind_address = format!("127.0.0.1:{port}");
rust/src/server/src/config.rs
+61
@@ -10,6 +10,10 @@ use serde_json::Value;
10 use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
11 use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
12
13 +/// Default keep-alive idle timeout (seconds); also the head-read bound
14 +/// when keep-alive is disabled (`0`).
15 +pub const DEFAULT_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5);
16 +
17 /// How the HTTP server obtains its listening socket.
18 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
19 pub enum HttpListenerMode {
@@ -99,6 +103,54 @@ impl CorsConfig {
103 }
104 }
105
106 +/// TLS settings mirroring Python's uvicorn `ssl_*` arguments.
107 +#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
108 +pub struct TlsConfig {
109 + /// PEM certificate chain file. Required when TLS is configured; may also
110 + /// hold the private key (combined PEM) when `key_file` is unset.
111 + pub cert_file: Option<String>,
112 + /// PEM private key file. When `None`, the key is read from `cert_file`
113 + /// (combined PEM).
114 + pub key_file: Option<String>,
115 + /// PEM CA bundle used to verify client certificates (mTLS). Required when
116 + /// `cert_reqs` is non-zero.
117 + pub ca_certs: Option<String>,
118 + /// Client-certificate requirement, mirroring Python's `ssl.CERT_*`:
119 + /// 0 = none, 1 = optional, 2 = required.
120 + pub cert_reqs: i32,
121 + /// OpenSSL cipher string for TLS 1.2 and below, mirroring Python's
122 + /// `ssl.set_ciphers`. `None` keeps the forward-secret AEAD default.
123 + pub ciphers: Option<String>,
124 +}
125 +
126 +impl TlsConfig {
127 + /// Structurally validate the TLS arguments; the cert/key material is parsed
128 + /// later, when the OpenSSL context is built.
129 + pub fn validate(&self) -> Result<()> {
130 + if self.cert_file.is_none() {
131 + bail!(
132 + "--ssl-certfile is required to enable TLS; \
133 + --ssl-keyfile/--ssl-ca-certs/--ssl-cert-reqs/--ssl-ciphers \
134 + cannot be used without it"
135 + );
136 + }
137 + if !matches!(self.cert_reqs, 0..=2) {
138 + bail!(
139 + "--ssl-cert-reqs must be 0 (none), 1 (optional), or 2 (required), got {}",
140 + self.cert_reqs
141 + );
142 + }
143 + if self.cert_reqs != 0 && self.ca_certs.is_none() {
144 + bail!(
145 + "--ssl-ca-certs is required when --ssl-cert-reqs is {} \
146 + (client certificate verification)",
147 + self.cert_reqs
148 + );
149 + }
150 + Ok(())
151 + }
152 +}
153 +
154 /// Normalized runtime configuration for the minimal OpenAI-compatible server.
155 #[derive(Educe, Clone, PartialEq, Eq, Serialize)]
156 #[educe(Debug)]
@@ -138,6 +190,9 @@ pub struct Config {
190 pub api_server_options: ApiServerOptions,
191 /// CORS settings applied to every HTTP response.
192 pub cors: CorsConfig,
193 + /// TLS settings. `None` serves plaintext HTTP; `Some` terminates TLS at the
194 + /// listener.
195 + pub tls: Option<TlsConfig>,
196 /// API keys accepted as bearer tokens for guarded routes.
197 #[serde(skip_serializing)]
198 #[educe(Debug(method(fmt_redacted_api_keys)))]
@@ -150,6 +205,9 @@ pub struct Config {
205 pub grpc_port: Option<u16>,
206 /// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
207 pub shutdown_timeout: Duration,
208 + /// Maximum idle time on a keep-alive HTTP connection before the server
209 + /// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s).
210 + pub keep_alive_timeout: Duration,
211 }
212
213 impl Config {
@@ -158,6 +216,9 @@ impl Config {
216 pub fn validate(&self) -> Result<()> {
217 vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
218 self.cors.validate()?;
219 + if let Some(tls) = &self.tls {
220 + tls.validate()?;
221 + }
222 if let Some(max_logprobs) = self.max_logprobs
223 && max_logprobs < -1
224 {
rust/src/server/src/grpc/mod.rs
+78 -1
@@ -4,16 +4,21 @@ mod convert;
4
5 use std::pin::Pin;
6 use std::sync::Arc;
7 +use std::task::{Context, Poll};
8
8 -use futures::{Stream, StreamExt as _};
9 +use futures::{Stream, StreamExt as _, stream};
10 use thiserror_ext::AsReport as _;
11 +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
12 use tokio::sync::mpsc;
13 +use tokio_openssl::SslStream;
14 use tokio_stream::wrappers::ReceiverStream;
15 +use tonic::transport::server::{Connected, TcpConnectInfo};
16 use tonic::{Request, Response, Status};
17 use tracing::info;
18 use vllm_text::{DecodedTextEvent, TextOutputStreamExt as _};
19
20 use self::convert::ResponseOpts;
21 +use crate::listener::{Listener, ListenerIo};
22 use crate::state::AppState;
23
24 /// Generated protobuf/gRPC types for the `vllm` package.
@@ -26,6 +31,78 @@ pub use pb::generate_server::GenerateServer;
31 #[cfg(test)]
32 mod tests;
33
34 +/// Newtype over `tokio-openssl`'s `SslStream` so we can implement tonic's
35 +/// [`Connected`] on it (the orphan rule blocks doing so on the foreign type).
36 +pub(crate) struct GrpcTlsStream {
37 + inner: SslStream<ListenerIo>,
38 +}
39 +
40 +impl GrpcTlsStream {
41 + pub(crate) fn new(inner: SslStream<ListenerIo>) -> Self {
42 + Self { inner }
43 + }
44 +}
45 +
46 +impl AsyncRead for GrpcTlsStream {
47 + fn poll_read(
48 + self: Pin<&mut Self>,
49 + cx: &mut Context<'_>,
50 + buf: &mut ReadBuf<'_>,
51 + ) -> Poll<std::io::Result<()>> {
52 + Pin::new(&mut self.get_mut().inner).poll_read(cx, buf)
53 + }
54 +}
55 +
56 +impl AsyncWrite for GrpcTlsStream {
57 + fn poll_write(
58 + self: Pin<&mut Self>,
59 + cx: &mut Context<'_>,
60 + buf: &[u8],
61 + ) -> Poll<std::io::Result<usize>> {
62 + Pin::new(&mut self.get_mut().inner).poll_write(cx, buf)
63 + }
64 +
65 + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
66 + Pin::new(&mut self.get_mut().inner).poll_flush(cx)
67 + }
68 +
69 + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
70 + Pin::new(&mut self.get_mut().inner).poll_shutdown(cx)
71 + }
72 +}
73 +
74 +impl Connected for GrpcTlsStream {
75 + type ConnectInfo = TcpConnectInfo;
76 +
77 + fn connect_info(&self) -> TcpConnectInfo {
78 + self.inner.get_ref().connect_info()
79 + }
80 +}
81 +
82 +/// Adapt the shared server listener into tonic's incoming stream shape.
83 +pub(crate) fn incoming(listener: Listener) -> impl Stream<Item = std::io::Result<ListenerIo>> {
84 + stream::unfold(listener, |mut listener| async move {
85 + let (io, _) = axum::serve::Listener::accept(&mut listener).await;
86 + Some((Ok(io), listener))
87 + })
88 +}
89 +
90 +/// Wrap the gRPC listener so each accepted connection completes a TLS handshake
91 +/// before tonic serves it.
92 +pub(crate) fn tls_incoming(
93 + listener: Listener,
94 + context: openssl::ssl::SslContext,
95 + handshake_timeout: std::time::Duration,
96 +) -> impl Stream<Item = std::io::Result<GrpcTlsStream>> {
97 + tls_listener::builder(context)
98 + .handshake_timeout(handshake_timeout)
99 + .listen(listener)
100 + .map(|res| {
101 + res.map(|(inner, _addr)| GrpcTlsStream::new(inner))
102 + .map_err(std::io::Error::other)
103 + })
104 +}
105 +
106 /// gRPC Generate service implementation backed by the shared application state.
107 pub struct GenerateServiceImpl {
108 state: Arc<AppState>,
rust/src/server/src/grpc/tests.rs
+357 -15
@@ -1,11 +1,19 @@
1 use std::future::Future;
2 +use std::io;
3 use std::pin::Pin;
4 use std::sync::Arc;
5 use std::task::{Context, Poll};
6 +use std::time::Duration;
7
8 use futures::StreamExt as _;
9 +use hyper_util::rt::TokioIo;
10 +use openssl::ssl::{SslConnector, SslFiletype, SslMethod};
11 use serial_test::serial;
8 -use tonic::transport::Server as TonicServer;
12 +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
13 +use tokio::net::TcpStream;
14 +use tokio_openssl::SslStream;
15 +use tonic::transport::{Channel, Endpoint, Server as TonicServer, Uri};
16 +use tower::service_fn;
17 use vllm_chat::{
18 ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
19 DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
@@ -22,8 +30,11 @@ use zeromq::prelude::{SocketRecv, SocketSend};
30 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
31
32 use super::pb::generate_client::GenerateClient;
25 -use super::{GenerateServer, GenerateServiceImpl, pb};
33 +use super::{GenerateServer, GenerateServiceImpl, incoming, pb, tls_incoming};
34 +use crate::listener::Listener;
35 use crate::state::AppState;
36 +use crate::tls;
37 +use crate::tls_tests::{TestCerts, server_tls};
38
39 // ========================================================================================
40 // Helpers (mirrors the patterns in routes/tests.rs)
@@ -211,17 +222,12 @@ impl ChatRenderer for FakeTextBackend {
222 }
223 }
224
214 -/// Spin up a gRPC server backed by a mock engine that serves a single request
215 -/// with the given output specs. Returns the client, the gRPC server task, and
216 -/// the mock engine task.
217 -async fn grpc_test_server(
225 +/// Build the gRPC service + mock engine that serves a single request with the
226 +/// given output specs. Shared by the plaintext and TLS server fixtures.
227 +async fn setup_grpc_service(
228 engine_id: impl Into<EngineId>,
229 output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
220 -) -> (
221 - GenerateClient<tonic::transport::Channel>,
222 - tokio::task::JoinHandle<()>,
223 - MockEngineTask,
224 -) {
230 +) -> (GenerateServer<GenerateServiceImpl>, MockEngineTask) {
231 let ipc = IpcNamespace::new().expect("create ipc namespace");
232 let handshake_address = ipc.handshake_endpoint();
233 let engine_id = engine_id.into();
@@ -259,14 +265,29 @@ async fn grpc_test_server(
265 Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
266 );
267 let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
262 - let svc = GenerateServer::new(GenerateServiceImpl::new(state));
268 + (
269 + GenerateServer::new(GenerateServiceImpl::new(state)),
270 + engine_task,
271 + )
272 +}
273 +
274 +/// Spin up a plaintext gRPC server backed by a mock engine. Returns the client,
275 +/// the gRPC server task, and the mock engine task.
276 +async fn grpc_test_server(
277 + engine_id: impl Into<EngineId>,
278 + output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
279 +) -> (
280 + GenerateClient<tonic::transport::Channel>,
281 + tokio::task::JoinHandle<()>,
282 + MockEngineTask,
283 +) {
284 + let (svc, engine_task) = setup_grpc_service(engine_id, output_specs).await;
285
264 - // Bind to an OS-assigned port.
286 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
287 let addr = listener.local_addr().expect("local addr");
288
289 let server_task = tokio::spawn(async move {
269 - let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener);
290 + let incoming = incoming(Listener::Tcp(listener));
291 TonicServer::builder()
292 .add_service(svc)
293 .serve_with_incoming(incoming)
@@ -274,7 +295,6 @@ async fn grpc_test_server(
295 .expect("grpc server");
296 });
297
277 - // Connect the client.
298 let grpc_client = GenerateClient::connect(format!("http://{addr}"))
299 .await
300 .expect("connect grpc client");
@@ -282,6 +302,158 @@ async fn grpc_test_server(
302 (grpc_client, server_task, engine_task)
303 }
304
305 +/// Spin up a TLS gRPC server (server cert from `certs`, `cert_reqs` mTLS mode).
306 +/// Returns the address, the server task, and the mock engine task.
307 +async fn grpc_tls_test_server(
308 + engine_id: impl Into<EngineId>,
309 + output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
310 + certs: &TestCerts,
311 + cert_reqs: i32,
312 +) -> (String, tokio::task::JoinHandle<()>, MockEngineTask) {
313 + let (svc, engine_task) = setup_grpc_service(engine_id, output_specs).await;
314 + let context = tls::build_grpc_server_config(&server_tls(certs, cert_reqs))
315 + .expect("build grpc tls config");
316 +
317 + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
318 + let addr = listener.local_addr().expect("local addr").to_string();
319 +
320 + let server_task = tokio::spawn(async move {
321 + let incoming = tls_incoming(Listener::Tcp(listener), context, tls::TLS_HANDSHAKE_TIMEOUT);
322 + TonicServer::builder()
323 + .add_service(svc)
324 + .serve_with_incoming(incoming)
325 + .await
326 + .expect("grpc tls server");
327 + });
328 +
329 + (addr, server_task, engine_task)
330 +}
331 +
332 +/// Build a tonic `Generate` client over a tokio-openssl connector, optionally
333 +/// with a client identity for mTLS. Hand-rolled because tonic 0.14 ships no
334 +/// OpenSSL transport.
335 +async fn grpc_tls_client(
336 + certs: &TestCerts,
337 + addr: &str,
338 + identity: Option<&str>,
339 +) -> Result<GenerateClient<Channel>, tonic::transport::Error> {
340 + let ca = certs.path("ca.pem");
341 + let identity = identity.map(|name| {
342 + (
343 + certs.path(&format!("{name}.pem")),
344 + certs.path(&format!("{name}.key")),
345 + )
346 + });
347 + let target = addr.to_string();
348 +
349 + let connector = service_fn(move |_: Uri| {
350 + let ca = ca.clone();
351 + let identity = identity.clone();
352 + let target = target.clone();
353 + async move {
354 + let tcp = TcpStream::connect(&target).await?;
355 + let mut builder =
356 + SslConnector::builder(SslMethod::tls_client()).map_err(io::Error::other)?;
357 + builder.set_ca_file(&ca).map_err(io::Error::other)?;
358 + if let Some((cert, key)) = &identity {
359 + builder.set_certificate_chain_file(cert).map_err(io::Error::other)?;
360 + builder.set_private_key_file(key, SslFiletype::PEM).map_err(io::Error::other)?;
361 + }
362 + let mut config = builder.build().configure().map_err(io::Error::other)?;
363 + config.set_verify_hostname(false);
364 + config.set_alpn_protos(b"\x02h2").map_err(io::Error::other)?;
365 + let ssl = config.into_ssl("127.0.0.1").map_err(io::Error::other)?;
366 + let mut stream = SslStream::new(ssl, tcp).map_err(io::Error::other)?;
367 + Pin::new(&mut stream).connect().await.map_err(io::Error::other)?;
368 + Ok::<_, io::Error>(TokioIo::new(stream))
369 + }
370 + });
371 +
372 + let channel = Endpoint::from_shared(format!("https://{addr}"))
373 + .expect("grpc endpoint")
374 + .connect_with_connector(connector)
375 + .await?;
376 + Ok(GenerateClient::new(channel))
377 +}
378 +
379 +/// Complete a raw TLS handshake against the gRPC port (offering ALPN `h2`) for
380 +/// the ALPN-negotiation assertion.
381 +async fn grpc_tls_handshake(
382 + certs: &TestCerts,
383 + addr: &str,
384 +) -> io::Result<Pin<Box<SslStream<TcpStream>>>> {
385 + let tcp = TcpStream::connect(addr).await?;
386 + let mut builder = SslConnector::builder(SslMethod::tls_client()).map_err(io::Error::other)?;
387 + builder.set_ca_file(certs.path("ca.pem")).map_err(io::Error::other)?;
388 + let mut config = builder.build().configure().map_err(io::Error::other)?;
389 + config.set_verify_hostname(false);
390 + config.set_alpn_protos(b"\x02h2").map_err(io::Error::other)?;
391 + let ssl = config.into_ssl("127.0.0.1").map_err(io::Error::other)?;
392 + let mut stream = Box::pin(SslStream::new(ssl, tcp).map_err(io::Error::other)?);
393 + stream.as_mut().connect().await.map_err(io::Error::other)?;
394 + Ok(stream)
395 +}
396 +
397 +/// Spin up a plaintext gRPC server, optionally with HTTP/2 keepalive set to
398 +/// `keepalive` for both the PING interval and the unanswered-PING timeout.
399 +async fn grpc_server_with_keepalive(
400 + engine_id: impl Into<EngineId>,
401 + keepalive: Option<Duration>,
402 +) -> (String, tokio::task::JoinHandle<()>, MockEngineTask) {
403 + let (svc, engine_task) = setup_grpc_service(engine_id, default_stream_output_specs()).await;
404 +
405 + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
406 + let addr = listener.local_addr().expect("local addr").to_string();
407 +
408 + let mut builder = TonicServer::builder();
409 + if let Some(interval) = keepalive {
410 + builder = builder
411 + .http2_keepalive_interval(Some(interval))
412 + .http2_keepalive_timeout(Some(interval));
413 + }
414 +
415 + let server_task = tokio::spawn(async move {
416 + let incoming = incoming(Listener::Tcp(listener));
417 + builder
418 + .add_service(svc)
419 + .serve_with_incoming(incoming)
420 + .await
421 + .expect("grpc server");
422 + });
423 +
424 + (addr, server_task, engine_task)
425 +}
426 +
427 +/// Establish an HTTP/2 connection (preface + SETTINGS exchange) then go silent,
428 +/// ACKing the server's SETTINGS but never its keepalive PINGs. Returns whether
429 +/// the SERVER closes the connection within `wait`. A minimal hand-rolled h2 peer
430 +/// because a real client auto-ACKs PINGs and so can never be kept-alive-evicted.
431 +async fn h2_unresponsive_peer_closed_within(addr: &str, wait: Duration) -> bool {
432 + let mut tcp = TcpStream::connect(addr).await.expect("connect");
433 + tcp.write_all(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n").await.expect("preface");
434 + tcp.write_all(&[0, 0, 0, 0x4, 0, 0, 0, 0, 0]).await.expect("client settings");
435 +
436 + let closed = tokio::time::timeout(wait, async {
437 + let mut header = [0u8; 9];
438 + while tcp.read_exact(&mut header).await.is_ok() {
439 + let len = u32::from_be_bytes([0, header[0], header[1], header[2]]) as usize;
440 + let frame_type = header[3];
441 + let flags = header[4];
442 + let mut payload = vec![0u8; len];
443 + if tcp.read_exact(&mut payload).await.is_err() {
444 + return;
445 + }
446 + // ACK the server's SETTINGS so the only thing left unanswered is PINGs.
447 + if frame_type == 0x4 && flags & 0x1 == 0 {
448 + let _ = tcp.write_all(&[0, 0, 0, 0x4, 0x1, 0, 0, 0, 0]).await;
449 + }
450 + }
451 + })
452 + .await;
453 +
454 + closed.is_ok()
455 +}
456 +
457 // ========================================================================================
458 // Tests
459 // ========================================================================================
@@ -720,3 +892,173 @@ async fn unary_generate_output_text_defaults_to_true() {
892 engine_task.await.expect("mock engine task");
893 server_task.abort();
894 }
895 +
896 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
897 +#[serial]
898 +async fn grpc_generate_succeeds_over_tls() {
899 + let certs = TestCerts::generate();
900 + let (addr, server_task, engine_task) = grpc_tls_test_server(
901 + b"engine-grpc-tls-unary",
902 + default_stream_output_specs(),
903 + &certs,
904 + 0,
905 + )
906 + .await;
907 +
908 + let mut client = grpc_tls_client(&certs, &addr, None).await.expect("tls client");
909 + let response = client
910 + .generate(pb::GenerateRequest {
911 + request_id: "test-tls-unary".to_string(),
912 + model: "test-model".to_string(),
913 + prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
914 + stopping: Some(pb::StoppingCriteria {
915 + max_new_tokens: 10,
916 + ..Default::default()
917 + }),
918 + response: Some(pb::ResponseOptions {
919 + output_text: Some(true),
920 + ..Default::default()
921 + }),
922 + ..Default::default()
923 + })
924 + .await
925 + .expect("unary generate over tls")
926 + .into_inner();
927 +
928 + assert_eq!(response.outputs.expect("outputs present").text, "hi");
929 +
930 + engine_task.await.expect("mock engine task");
931 + server_task.abort();
932 +}
933 +
934 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
935 +#[serial]
936 +async fn grpc_tls_negotiates_h2_alpn() {
937 + let certs = TestCerts::generate();
938 + let (addr, server_task, _engine_task) = grpc_tls_test_server(
939 + b"engine-grpc-tls-alpn",
940 + default_stream_output_specs(),
941 + &certs,
942 + 0,
943 + )
944 + .await;
945 +
946 + let stream = grpc_tls_handshake(&certs, &addr).await.expect("handshake");
947 + assert_eq!(
948 + stream.ssl().selected_alpn_protocol(),
949 + Some(&b"h2"[..]),
950 + "server must negotiate h2 ALPN"
951 + );
952 +
953 + server_task.abort();
954 +}
955 +
956 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
957 +#[serial]
958 +async fn grpc_mtls_required_rejects_client_without_certificate() {
959 + let certs = TestCerts::generate();
960 + let (addr, server_task, _engine_task) = grpc_tls_test_server(
961 + b"engine-grpc-tls-mtls-reject",
962 + default_stream_output_specs(),
963 + &certs,
964 + 2,
965 + )
966 + .await;
967 +
968 + // With TLS 1.3 the missing-client-cert rejection surfaces on first use, not
969 + // at the handshake, so drive an RPC and assert the call fails.
970 + let outcome = match grpc_tls_client(&certs, &addr, None).await {
971 + Err(_) => Err(()),
972 + Ok(mut client) => client
973 + .generate(pb::GenerateRequest {
974 + request_id: "test-tls-mtls-reject".to_string(),
975 + model: "test-model".to_string(),
976 + prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
977 + stopping: Some(pb::StoppingCriteria {
978 + max_new_tokens: 10,
979 + ..Default::default()
980 + }),
981 + ..Default::default()
982 + })
983 + .await
984 + .map(|_| ())
985 + .map_err(|_| ()),
986 + };
987 + assert!(
988 + outcome.is_err(),
989 + "mTLS-required gRPC must reject a client without a certificate"
990 + );
991 +
992 + server_task.abort();
993 +}
994 +
995 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
996 +#[serial]
997 +async fn grpc_mtls_required_accepts_valid_client_certificate() {
998 + let certs = TestCerts::generate();
999 + let (addr, server_task, engine_task) = grpc_tls_test_server(
1000 + b"engine-grpc-tls-mtls-accept",
1001 + default_stream_output_specs(),
1002 + &certs,
1003 + 2,
1004 + )
1005 + .await;
1006 +
1007 + let mut client = grpc_tls_client(&certs, &addr, Some("client")).await.expect("mtls client");
1008 + let response = client
1009 + .generate(pb::GenerateRequest {
1010 + request_id: "test-tls-mtls".to_string(),
1011 + model: "test-model".to_string(),
1012 + prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
1013 + stopping: Some(pb::StoppingCriteria {
1014 + max_new_tokens: 10,
1015 + ..Default::default()
1016 + }),
1017 + response: Some(pb::ResponseOptions {
1018 + output_text: Some(true),
1019 + ..Default::default()
1020 + }),
1021 + ..Default::default()
1022 + })
1023 + .await
1024 + .expect("mtls generate over tls")
1025 + .into_inner();
1026 +
1027 + assert_eq!(response.outputs.expect("outputs present").text, "hi");
1028 +
1029 + engine_task.await.expect("mock engine task");
1030 + server_task.abort();
1031 +}
1032 +
1033 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1034 +#[serial]
1035 +async fn grpc_keepalive_closes_unresponsive_connection() {
1036 + let (addr, server_task, _engine_task) =
1037 + grpc_server_with_keepalive(b"engine-grpc-keepalive", Some(Duration::from_millis(150)))
1038 + .await;
1039 +
1040 + let closed = h2_unresponsive_peer_closed_within(&addr, Duration::from_secs(5)).await;
1041 + assert!(
1042 + closed,
1043 + "keepalive must close a peer that stops answering PINGs"
1044 + );
1045 +
1046 + server_task.abort();
1047 +}
1048 +
1049 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1050 +#[serial]
1051 +async fn grpc_without_keepalive_keeps_unresponsive_connection_open() {
1052 + // Without keepalive the same unresponsive peer is NOT
1053 + // closed, proving the close above is attributable to keepalive.
1054 + let (addr, server_task, _engine_task) =
1055 + grpc_server_with_keepalive(b"engine-grpc-no-keepalive", None).await;
1056 +
1057 + let closed = h2_unresponsive_peer_closed_within(&addr, Duration::from_secs(1)).await;
1058 + assert!(
1059 + !closed,
1060 + "without keepalive an idle h2 connection must stay open"
1061 + );
1062 +
1063 + server_task.abort();
1064 +}
rust/src/server/src/lib.rs
+205 -35
@@ -10,20 +10,34 @@ mod routes;
10 mod runtime;
11 mod server_info;
12 mod state;
13 +mod tls;
14 +#[cfg(test)]
15 +mod tls_tests;
16 mod utils;
17
18 +use std::future::Future;
19 use std::sync::{Arc, OnceLock};
20 +use std::time::Duration;
21
22 use anyhow::{Context as _, Result};
23 use axum::Router;
19 -use axum::serve::ListenerExt as _;
20 -pub use config::{ApiServerOptions, Config, CoordinatorMode, CorsConfig, HttpListenerMode};
24 +use axum::body::Body;
25 +use axum::http::Request;
26 +pub use config::{
27 + ApiServerOptions, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT,
28 + HttpListenerMode, TlsConfig,
29 +};
30 +use futures::FutureExt as _;
31 +use hyper::body::Incoming;
32 +use hyper::server::conn::http1;
33 +use hyper_util::rt::{TokioIo, TokioTimer};
34 +use hyper_util::server::graceful::GracefulShutdown;
35 +use hyper_util::service::TowerToHyperService;
36 use tokio::net::TcpListener;
37 use tokio::time::{Instant, sleep_until};
23 -use tokio_stream::wrappers::TcpListenerStream;
24 -use tokio_util::either::Either;
38 use tokio_util::sync::CancellationToken;
39 use tonic::transport::Server as TonicServer;
40 +use tower::ServiceExt as _;
41 use tracing::{info, trace, warn};
42 use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};
43 pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
@@ -36,6 +50,13 @@ use crate::routes::build_router;
50 use crate::server_info::ServerInfoSnapshot;
51 use crate::state::AppState;
52
53 +/// How often the server PINGs an idle gRPC connection to reap a dead peer;
54 +/// tonic enables no keepalive by default. 2h matches the gRPC-core default.
55 +const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(7200);
56 +/// How long the server waits for a keepalive PING reply before dropping the gRPC
57 +/// connection. 20s matches the gRPC-core default.
58 +const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20);
59 +
60 /// Resolve the public model names accepted by the frontend.
61 fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Vec<String> {
62 if served_model_name.is_empty() {
@@ -45,6 +66,17 @@ fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Ve
66 }
67 }
68
69 +/// Choose the gRPC listener host. It follows the HTTP TCP host when there is
70 +/// one; otherwise (unix socket or inherited fd) it defaults to IPv4 loopback
71 +/// rather than all interfaces, so the side-car is never accidentally
72 +/// network-exposed.
73 +fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
74 + match listener_mode {
75 + HttpListenerMode::BindTcp { host, .. } => host.as_str(),
76 + HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "127.0.0.1",
77 + }
78 +}
79 +
80 /// Build the shared application state for one configured model and one engine
81 /// client.
82 async fn build_state(config: &Config) -> Result<Arc<AppState>> {
@@ -130,6 +162,15 @@ where
162 {
163 config.validate().context("invalid OpenAI frontend configuration")?;
164
165 + // Build the TLS server config once, up front, so a bad cert/key fails fast
166 + // before the (potentially long) engine handshake.
167 + let tls_config = config
168 + .tls
169 + .as_ref()
170 + .map(tls::build_server_config)
171 + .transpose()
172 + .context("invalid TLS configuration")?;
173 +
174 // Also check shutdown during the (potentially long) startup handshake.
175 let state = tokio::select! {
176 result = build_state(&config) => result?,
@@ -144,40 +185,39 @@ where
185
186 // Optionally bind the gRPC Generate server on a separate port. Bind
187 // synchronously here so bind errors (port in use, permission denied, ...)
147 - // surface before we start serving, rather than being deferred until
148 - // shutdown. The gRPC listener follows the same host as the HTTP listener so
149 - // that enabling --grpc-port does not accidentally expose the service on all
150 - // interfaces when HTTP is intentionally local-only.
188 + // surface before serving rather than being deferred until shutdown.
189 let grpc_setup = if let Some(grpc_port) = config.grpc_port {
152 - let grpc_host = match &config.listener_mode {
153 - HttpListenerMode::BindTcp { host, .. } => host.as_str(),
154 - HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "0.0.0.0",
155 - };
190 + let grpc_host = grpc_bind_host(&config.listener_mode);
191 let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
192 .await
193 .with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
194 let addr = grpc_listener.local_addr()?;
195 + let grpc_listener = Listener::Tcp(grpc_listener);
196 + // gRPC reuses the HTTP TLS config (same SslContext) plus ALPN h2.
197 + let grpc_tls = config
198 + .tls
199 + .as_ref()
200 + .map(tls::build_grpc_server_config)
201 + .transpose()
202 + .context("invalid gRPC TLS configuration")?;
203 let svc = grpc::GenerateServer::new(grpc::GenerateServiceImpl::new(state.clone()));
204 let svc = TonicServer::builder()
205 + .http2_keepalive_interval(Some(GRPC_KEEPALIVE_INTERVAL))
206 + .http2_keepalive_timeout(Some(GRPC_KEEPALIVE_TIMEOUT))
207 .layer(middleware::request_runtime_layer(state.clone()))
208 .add_service(svc);
164 - info!(%addr, "starting gRPC server");
165 - Some((grpc_listener, svc))
209 + info!(%addr, tls = grpc_tls.is_some(), "starting gRPC server");
210 + Some((grpc_listener, svc, grpc_tls))
211 } else {
212 None
213 };
214
170 - info!(%bind_address, %model, "starting OpenAI server");
171 -
172 - // Set TCP_NODELAY on accepted connections to reduce latency.
173 - // By `tap_io` we will do this on every accepted connection.
174 - let listener = listener.tap_io(|io| {
175 - if let Either::Left(tcp_stream) = io
176 - && let Err(err) = tcp_stream.set_nodelay(true)
177 - {
178 - trace!(error = %err, "failed to enable TCP_NODELAY on accepted HTTP connection");
179 - }
180 - });
215 + let scheme = if tls_config.is_some() {
216 + "https"
217 + } else {
218 + "http"
219 + };
220 + info!(%bind_address, %scheme, %model, "starting OpenAI server");
221
222 // Run HTTP and gRPC concurrently under a child token of the caller's shutdown
223 // token. Caller cancellation propagates into both protocols; if either
@@ -208,17 +248,27 @@ where
248 }
249 });
250
251 + // 0 disables keep-alive but still bounds the head read (default), so a
252 + // silent client cannot hold the connection open.
253 + let keep_alive_timeout = config.keep_alive_timeout;
254 + let timeouts = ConnectionTimeouts {
255 + handshake: tls::TLS_HANDSHAKE_TIMEOUT,
256 + header_read: if keep_alive_timeout.is_zero() {
257 + DEFAULT_KEEP_ALIVE_TIMEOUT
258 + } else {
259 + keep_alive_timeout
260 + },
261 + keep_alive_enabled: !keep_alive_timeout.is_zero(),
262 + };
263 +
264 let http_fut = {
265 let shutdown = server_shutdown.child_token();
266 let server_shutdown = server_shutdown.clone();
267 let force_shutdown = force_shutdown.clone();
268 async move {
216 - let server =
217 - axum::serve(listener, app).with_graceful_shutdown(shutdown.cancelled_owned());
218 -
269 let result = tokio::select! {
220 - result = server => {
221 - result.context("HTTP server failed")
270 + result = serve_listener(listener, tls_config, app, shutdown.cancelled_owned(), timeouts) => {
271 + result
272 }
273 _ = force_shutdown.cancelled() => {
274 warn!("HTTP graceful shutdown deadline elapsed; aborting server");
@@ -236,16 +286,24 @@ where
286 let server_shutdown = server_shutdown.clone();
287 let force_shutdown = force_shutdown.clone();
288 async move {
239 - let Some((grpc_listener, svc)) = grpc_setup else {
289 + let Some((grpc_listener, svc, grpc_tls)) = grpc_setup else {
290 // No gRPC configured: just wait for shutdown so we do not race the
291 // join! by resolving early and tripping the cancellation token.
292 shutdown.cancelled().await;
293 return Ok(());
294 };
245 - let server = svc.serve_with_incoming_shutdown(
246 - TcpListenerStream::new(grpc_listener),
247 - shutdown.cancelled_owned(),
248 - );
295 + // Box to unify the TLS and plaintext arms' different stream types.
296 + let server = match grpc_tls {
297 + Some(context) => {
298 + let incoming =
299 + grpc::tls_incoming(grpc_listener, context, tls::TLS_HANDSHAKE_TIMEOUT);
300 + svc.serve_with_incoming_shutdown(incoming, shutdown.cancelled_owned()).boxed()
301 + }
302 + None => {
303 + let incoming = grpc::incoming(grpc_listener);
304 + svc.serve_with_incoming_shutdown(incoming, shutdown.cancelled_owned()).boxed()
305 + }
306 + };
307
308 let result = tokio::select! {
309 result = server => {
@@ -272,6 +330,99 @@ where
330 state.shutdown(shutdown_deadline).await
331 }
332
333 +/// Per-connection timeouts applied while serving HTTP/HTTPS.
334 +#[derive(Clone, Copy)]
335 +pub(crate) struct ConnectionTimeouts {
336 + /// Max time for a client to complete the TLS handshake (TLS path only).
337 + pub(crate) handshake: Duration,
338 + /// HTTP/1 header-read timeout (bounds idle keep-alive and the head read).
339 + pub(crate) header_read: Duration,
340 + /// Whether HTTP/1 keep-alive is enabled; `false` closes after each response.
341 + pub(crate) keep_alive_enabled: bool,
342 +}
343 +
344 +/// Apply optional TLS termination and per-connection HTTP timeouts, then serve
345 +/// `app`. Shared by [`serve_with_router_extension`] and the TLS tests.
346 +async fn serve_listener(
347 + listener: Listener,
348 + tls: Option<openssl::ssl::SslContext>,
349 + app: Router,
350 + shutdown: impl Future<Output = ()> + Send + 'static,
351 + timeouts: ConnectionTimeouts,
352 +) -> Result<()> {
353 + match tls {
354 + Some(context) => {
355 + // tls-listener terminates TLS (handshake + timeout); serve_connections
356 + // owns the HTTP keep-alive/idle bound that axum::serve cannot express.
357 + // Failed handshakes (incl. timeouts) log at ERROR via tls-listener.
358 + let listener = tls_listener::builder(context)
359 + .handshake_timeout(timeouts.handshake)
360 + .listen(listener);
361 + serve_connections(
362 + listener,
363 + app,
364 + shutdown,
365 + timeouts.header_read,
366 + timeouts.keep_alive_enabled,
367 + )
368 + .await
369 + .context("HTTPS server failed")
370 + }
371 + None => serve_connections(
372 + listener,
373 + app,
374 + shutdown,
375 + timeouts.header_read,
376 + timeouts.keep_alive_enabled,
377 + )
378 + .await
379 + .context("HTTP server failed"),
380 + }
381 +}
382 +
383 +/// Serve `app` per connection (HTTP/1) with a keep-alive idle timeout and
384 +/// graceful drain. Hand-rolled on hyper because [`axum::serve()`] takes no config.
385 +async fn serve_connections<L>(
386 + mut listener: L,
387 + app: Router,
388 + shutdown: impl Future<Output = ()> + Send,
389 + header_read: Duration,
390 + keep_alive_enabled: bool,
391 +) -> Result<()>
392 +where
393 + L: axum::serve::Listener,
394 +{
395 + let graceful = GracefulShutdown::new();
396 + let mut shutdown = std::pin::pin!(shutdown);
397 + loop {
398 + let (io, _addr) = tokio::select! {
399 + conn = listener.accept() => conn,
400 + () = &mut shutdown => break,
401 + };
402 +
403 + let service = TowerToHyperService::new(
404 + app.clone().map_request(|req: Request<Incoming>| req.map(Body::new)),
405 + );
406 + let mut builder = http1::Builder::new();
407 + builder.timer(TokioTimer::new()).header_read_timeout(header_read);
408 + if !keep_alive_enabled {
409 + builder.keep_alive(false);
410 + }
411 + let connection = builder.serve_connection(TokioIo::new(io), service);
412 + let connection = graceful.watch(connection);
413 +
414 + tokio::spawn(async move {
415 + if let Err(err) = connection.await {
416 + trace!(error = %err, "failed to serve connection");
417 + }
418 + });
419 + }
420 +
421 + drop(listener);
422 + graceful.shutdown().await;
423 + Ok(())
424 +}
425 +
426 #[cfg(test)]
427 mod tests {
428 use super::*;
@@ -293,4 +444,23 @@ mod tests {
444 served_names
445 );
446 }
447 +
448 + #[test]
449 + fn grpc_bind_host_follows_http_tcp_host() {
450 + let mode = HttpListenerMode::BindTcp {
451 + host: "0.0.0.0".to_string(),
452 + port: 8000,
453 + };
454 + assert_eq!(grpc_bind_host(&mode), "0.0.0.0");
455 + }
456 +
457 + #[test]
458 + fn grpc_bind_host_defaults_to_loopback_without_tcp_host() {
459 + let unix = HttpListenerMode::BindUnix {
460 + path: "/tmp/vllm.sock".to_string(),
461 + };
462 + let inherited = HttpListenerMode::InheritedFd { fd: 3 };
463 + assert_eq!(grpc_bind_host(&unix), "127.0.0.1");
464 + assert_eq!(grpc_bind_host(&inherited), "127.0.0.1");
465 + }
466 }
rust/src/server/src/listener.rs
+96 -14
@@ -1,28 +1,49 @@
1 -//! Unified HTTP listener wrapper for the Rust frontend.
1 +//! Unified listener wrapper for the Rust frontend.
2 //!
3 //! This module hides the difference between TCP and Unix-domain listeners so
4 //! the rest of the server can bind or inherit one socket and pass it to
5 //! `axum::serve(...)` through a single type.
6
7 use std::io::Result;
8 -use std::net::TcpListener as StdTcpListener;
8 +use std::net::{SocketAddr, TcpListener as StdTcpListener};
9 use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
10 use std::os::unix::net::UnixListener as StdUnixListener;
11 +use std::pin::Pin;
12 +use std::task::{Context, Poll, ready};
13
14 +use auto_enums::enum_derive;
15 use socket2::Socket;
16 +use tls_listener::{AsyncAccept, AsyncListener};
17 use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream};
14 -use tokio_util::either::Either;
18 +use tonic::transport::server::{Connected, TcpConnectInfo};
19 +use tracing::trace;
20
21 use crate::HttpListenerMode;
22
18 -/// Runtime listener type used by the OpenAI-compatible HTTP server, which is
19 -/// either a TCP listener or a Unix-domain listener.
23 +/// Runtime listener type used by the OpenAI-compatible HTTP or gRPC server,
24 +/// which is either a TCP listener or a Unix-domain listener.
25 #[derive(Debug)]
26 pub enum Listener {
27 Tcp(TcpListener),
28 Unix(UnixListener),
29 }
30
31 +/// Runtime listener I/O type which is either a TCP stream or a Unix-domain stream.
32 +#[derive(Debug)]
33 +#[enum_derive(tokio1::AsyncRead, tokio1::AsyncWrite)]
34 +pub enum ListenerIo {
35 + Tcp(TcpStream),
36 + Unix(UnixStream),
37 +}
38 +
39 +/// Runtime listener address type which is either a TCP address or a Unix-domain address.
40 +#[derive(Debug)]
41 +#[allow(dead_code)]
42 +pub enum ListenerAddr {
43 + Tcp(SocketAddr),
44 + Unix(tokio::net::unix::SocketAddr),
45 +}
46 +
47 impl Listener {
48 /// Bind or adopt the listener described by the frontend configuration.
49 ///
@@ -70,34 +91,95 @@ impl Listener {
91 Ok(Self::Tcp(TcpListener::from_std(std_listener)?))
92 }
93 }
94 +
95 + fn listener_addr(&self) -> Result<ListenerAddr> {
96 + match self {
97 + Self::Tcp(listener) => listener.local_addr().map(ListenerAddr::Tcp),
98 + Self::Unix(listener) => listener.local_addr().map(ListenerAddr::Unix),
99 + }
100 + }
101 +}
102 +
103 +impl Connected for ListenerIo {
104 + type ConnectInfo = TcpConnectInfo;
105 +
106 + fn connect_info(&self) -> TcpConnectInfo {
107 + match self {
108 + Self::Tcp(stream) => stream.connect_info(),
109 + Self::Unix(_) => TcpConnectInfo {
110 + local_addr: None,
111 + remote_addr: None,
112 + },
113 + }
114 + }
115 +}
116 +
117 +/// Attempt to set `TCP_NODELAY` on the accepted TCP stream.
118 +fn enable_tcp_nodelay(stream: TcpStream) -> TcpStream {
119 + if let Err(err) = stream.set_nodelay(true) {
120 + trace!(error = %err, "failed to enable TCP_NODELAY on accepted TCP connection");
121 + }
122 + stream
123 }
124
125 /// Allow the unified listener to plug directly into `axum::serve(...)`.
126 impl axum::serve::Listener for Listener {
77 - type Addr = Either<std::net::SocketAddr, tokio::net::unix::SocketAddr>;
78 - type Io = Either<TcpStream, UnixStream>;
127 + type Addr = ListenerAddr;
128 + type Io = ListenerIo;
129
130 async fn accept(&mut self) -> (Self::Io, Self::Addr) {
131 match self {
132 Self::Tcp(listener) => {
83 - let (io, addr) = listener.accept().await;
84 - (Either::Left(io), Either::Left(addr))
133 + let (io, addr) = axum::serve::Listener::accept(listener).await;
134 + (
135 + ListenerIo::Tcp(enable_tcp_nodelay(io)),
136 + ListenerAddr::Tcp(addr),
137 + )
138 }
139 Self::Unix(listener) => {
87 - let (io, addr) = listener.accept().await;
88 - (Either::Right(io), Either::Right(addr))
140 + let (io, addr) = axum::serve::Listener::accept(listener).await;
141 + (ListenerIo::Unix(io), ListenerAddr::Unix(addr))
142 }
143 }
144 }
145
146 fn local_addr(&self) -> Result<Self::Addr> {
94 - match self {
95 - Self::Tcp(listener) => listener.local_addr().map(Either::Left),
96 - Self::Unix(listener) => listener.local_addr().map(Either::Right),
147 + self.listener_addr()
148 + }
149 +}
150 +
151 +/// Allow the unified listener to be adaptable to `tls_listener`.
152 +impl AsyncAccept for Listener {
153 + type Connection = ListenerIo;
154 + type Address = ListenerAddr;
155 + type Error = std::io::Error;
156 +
157 + fn poll_accept(
158 + self: Pin<&mut Self>,
159 + cx: &mut Context<'_>,
160 + ) -> Poll<Result<(Self::Connection, Self::Address)>> {
161 + match self.get_mut() {
162 + Self::Tcp(listener) => {
163 + let (io, addr) = ready!(listener.poll_accept(cx))?;
164 + Poll::Ready(Ok((
165 + ListenerIo::Tcp(enable_tcp_nodelay(io)),
166 + ListenerAddr::Tcp(addr),
167 + )))
168 + }
169 + Self::Unix(listener) => {
170 + let (io, addr) = ready!(listener.poll_accept(cx))?;
171 + Poll::Ready(Ok((ListenerIo::Unix(io), ListenerAddr::Unix(addr))))
172 + }
173 }
174 }
175 }
176
177 +impl AsyncListener for Listener {
178 + fn local_addr(&self) -> Result<Self::Address> {
179 + self.listener_addr()
180 + }
181 +}
182 +
183 #[cfg(test)]
184 mod tests {
185 use std::net::{Ipv4Addr, SocketAddrV4};
rust/src/server/src/tls.rs new
+119
@@ -0,0 +1,119 @@
1 +//! OpenSSL server-config construction for TLS termination.
2 +//!
3 +//! Builds an OpenSSL [`SslContext`] from the uvicorn-style `ssl_*` arguments
4 +//! (certificate chain, private key, mTLS client verifier, optional cipher list).
5 +//! The `tls-listener` crate drives the handshake on each accepted connection.
6 +//!
7 +//! Crypto runs through whichever OpenSSL the binary links (system by default,
8 +//! vendored when built with that feature).
9 +
10 +use std::path::Path;
11 +use std::time::Duration;
12 +
13 +use anyhow::{Context as _, Result};
14 +use openssl::ssl::{
15 + AlpnError, SslAcceptor, SslAcceptorBuilder, SslContext, SslContextBuilder, SslFiletype,
16 + SslMethod, SslOptions, SslVerifyMode, select_next_proto,
17 +};
18 +
19 +use crate::config::TlsConfig;
20 +
21 +/// Time a client has to complete the TLS handshake before the connection is dropped.
22 +pub(crate) const TLS_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(60);
23 +
24 +/// ALPN wire bytes for HTTP/2 (length-prefixed).
25 +const ALPN_H2: &[u8] = b"\x02h2";
26 +
27 +/// Build the shared OpenSSL acceptor from validated [`TlsConfig`]: the full
28 +/// certificate chain, the private key (`key_file`, or the certificate file when
29 +/// unset), the mTLS client verifier, and an optional cipher list.
30 +///
31 +/// Starts from the Mozilla intermediate baseline (forward-secret AEAD suites,
32 +/// TLS 1.2 floor, server cipher preference, no compression), a slightly
33 +/// stricter subset of the Python frontend's default suites; `--ssl-ciphers`
34 +/// overrides it.
35 +fn build_server_builder(tls: &TlsConfig) -> Result<SslAcceptorBuilder> {
36 + let cert_file = tls.cert_file.as_deref().context("--ssl-certfile is required to enable TLS")?;
37 +
38 + let mut builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls_server())
39 + .context("failed to initialize TLS")?;
40 + builder.set_options(SslOptions::CIPHER_SERVER_PREFERENCE);
41 +
42 + // Load the whole chain (leaf + intermediates), not just the leaf, so
43 + // deployments behind an intermediate CA serve a complete chain.
44 + ensure_exists(cert_file, "--ssl-certfile")?;
45 + builder.set_certificate_chain_file(cert_file).with_context(|| {
46 + format!("failed to parse certificate chain in --ssl-certfile {cert_file:?}")
47 + })?;
48 +
49 + // When `key_file` is unset the key is read from the certificate file
50 + // (combined PEM).
51 + let key_file = tls.key_file.as_deref().unwrap_or(cert_file);
52 + ensure_exists(key_file, "private key file")?;
53 + builder
54 + .set_private_key_file(key_file, SslFiletype::PEM)
55 + .with_context(|| format!("failed to parse private key in {key_file:?}"))?;
56 + builder
57 + .check_private_key()
58 + .context("the certificate and private key do not match")?;
59 +
60 + configure_client_auth(&mut builder, tls)?;
61 +
62 + if let Some(ciphers) = tls.ciphers.as_deref().filter(|c| !c.is_empty()) {
63 + builder
64 + .set_cipher_list(ciphers)
65 + .with_context(|| format!("invalid --ssl-ciphers {ciphers:?}"))?;
66 + }
67 +
68 + Ok(builder)
69 +}
70 +
71 +/// Build the HTTP [`SslContext`] (HTTP/1.1; no ALPN, matching uvicorn).
72 +pub(crate) fn build_server_config(tls: &TlsConfig) -> Result<SslContext> {
73 + Ok(build_server_builder(tls)?.build().into_context())
74 +}
75 +
76 +/// Build the gRPC [`SslContext`]: identical to [`build_server_config`] but
77 +/// negotiates ALPN `h2`, which HTTP/2 over TLS requires.
78 +pub(crate) fn build_grpc_server_config(tls: &TlsConfig) -> Result<SslContext> {
79 + let mut builder = build_server_builder(tls)?;
80 + builder.set_alpn_select_callback(|_ssl, client| {
81 + select_next_proto(ALPN_H2, client).ok_or(AlpnError::NOACK)
82 + });
83 + Ok(builder.build().into_context())
84 +}
85 +
86 +/// Fail loudly with a flag-named message when a configured file is missing,
87 +/// distinguishing it from a malformed-PEM error raised later by OpenSSL (whose
88 +/// `ErrorStack` does not name the offending file).
89 +fn ensure_exists(path: &str, what: &str) -> Result<()> {
90 + std::fs::metadata(Path::new(path))
91 + .map(drop)
92 + .with_context(|| format!("failed to read {what} {path:?}"))
93 +}
94 +
95 +/// Apply the `cert_reqs` client-certificate policy: 0 = none, 1 = optional
96 +/// (verify if presented, allow anonymous), 2 = required. `PEER` without a custom
97 +/// verify callback still rejects a presented-but-untrusted certificate.
98 +fn configure_client_auth(builder: &mut SslContextBuilder, tls: &TlsConfig) -> Result<()> {
99 + if tls.cert_reqs == 0 {
100 + builder.set_verify(SslVerifyMode::NONE);
101 + return Ok(());
102 + }
103 +
104 + let ca_file = tls
105 + .ca_certs
106 + .as_deref()
107 + .context("--ssl-ca-certs is required for client certificate verification")?;
108 + ensure_exists(ca_file, "--ssl-ca-certs")?;
109 + builder
110 + .set_ca_file(ca_file)
111 + .with_context(|| format!("failed to parse --ssl-ca-certs {ca_file:?}"))?;
112 +
113 + let mut mode = SslVerifyMode::PEER;
114 + if tls.cert_reqs == 2 {
115 + mode |= SslVerifyMode::FAIL_IF_NO_PEER_CERT;
116 + }
117 + builder.set_verify(mode);
118 + Ok(())
119 +}
rust/src/server/src/tls_tests.rs new
+688
@@ -0,0 +1,688 @@
1 +//! TLS tests: `build_server_config` unit checks plus end-to-end OpenSSL handshakes
2 +//! through the production `serve_listener` path, with a trivial router since TLS
3 +//! terminates below the app.
4 +
5 +use std::pin::Pin;
6 +use std::time::Duration;
7 +
8 +use axum::Router;
9 +use axum::routing::get;
10 +use openssl::asn1::Asn1Time;
11 +use openssl::bn::{BigNum, MsbOption};
12 +use openssl::ec::{EcGroup, EcKey};
13 +use openssl::hash::MessageDigest;
14 +use openssl::nid::Nid;
15 +use openssl::pkey::{PKey, Private};
16 +use openssl::ssl::{SslConnector, SslFiletype, SslMethod, SslVersion};
17 +use openssl::x509::extension::{BasicConstraints, KeyUsage, SubjectAlternativeName};
18 +use openssl::x509::{X509, X509NameBuilder};
19 +use tempfile::TempDir;
20 +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
21 +use tokio::net::TcpStream;
22 +use tokio_openssl::SslStream;
23 +use tokio_util::sync::CancellationToken;
24 +
25 +use crate::config::{HttpListenerMode, TlsConfig};
26 +use crate::listener::Listener;
27 +use crate::{ConnectionTimeouts, serve_listener, tls};
28 +
29 +// ============================================================================
30 +// Test infrastructure
31 +// ============================================================================
32 +
33 +/// A throwaway CA + server/client/untrusted/chain cert set as PEM files in a
34 +/// temp dir; dropping it deletes them.
35 +pub(crate) struct TestCerts {
36 + dir: TempDir,
37 +}
38 +
39 +impl TestCerts {
40 + pub(crate) fn generate() -> Self {
41 + let dir = tempfile::tempdir().expect("tempdir");
42 +
43 + let (ca, ca_key) = build_ca();
44 + let (server, server_key) = build_leaf("server", &["127.0.0.1", "localhost"], &ca, &ca_key);
45 + let (client, client_key) = build_leaf("client", &[], &ca, &ca_key);
46 + let (untrusted, untrusted_key) = build_self_signed("untrusted client");
47 +
48 + // Leaf signed by an intermediate (itself signed by the root); the cert
49 + // file holds leaf + intermediate, for the chain-serving test.
50 + let (intermediate, intermediate_key) = build_intermediate(&ca, &ca_key);
51 + let (chain_leaf, chain_leaf_key) = build_leaf(
52 + "chain",
53 + &["127.0.0.1", "localhost"],
54 + &intermediate,
55 + &intermediate_key,
56 + );
57 +
58 + let server_pem = pem(&server);
59 + let server_key_pem = key_pem(&server_key);
60 + let files = [
61 + ("ca.pem", pem(&ca)),
62 + ("server.pem", server_pem.clone()),
63 + ("server.key", server_key_pem.clone()),
64 + ("client.pem", pem(&client)),
65 + ("client.key", key_pem(&client_key)),
66 + ("untrusted_client.pem", pem(&untrusted)),
67 + ("untrusted_client.key", key_pem(&untrusted_key)),
68 + (
69 + "server_combined.pem",
70 + format!("{server_pem}{server_key_pem}"),
71 + ),
72 + (
73 + "server_chain.pem",
74 + format!("{}{}", pem(&chain_leaf), pem(&intermediate)),
75 + ),
76 + ("server_chain.key", key_pem(&chain_leaf_key)),
77 + ];
78 + for (name, contents) in files {
79 + std::fs::write(dir.path().join(name), contents).expect("write fixture");
80 + }
81 + Self { dir }
82 + }
83 +
84 + /// Absolute path to a fixture by name; the file need not exist.
85 + pub(crate) fn path(&self, name: &str) -> String {
86 + self.dir.path().join(name).to_str().expect("utf-8 path").to_string()
87 + }
88 +}
89 +
90 +fn gen_key() -> PKey<Private> {
91 + let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).expect("ec group");
92 + let ec = EcKey::generate(&group).expect("ec key");
93 + PKey::from_ec_key(ec).expect("pkey")
94 +}
95 +
96 +fn serial() -> openssl::asn1::Asn1Integer {
97 + let mut bn = BigNum::new().expect("bignum");
98 + bn.rand(159, MsbOption::MAYBE_ZERO, false).expect("rand serial");
99 + bn.to_asn1_integer().expect("asn1 serial")
100 +}
101 +
102 +fn x509_name(cn: &str) -> openssl::x509::X509Name {
103 + let mut builder = X509NameBuilder::new().expect("name builder");
104 + builder.append_entry_by_text("CN", cn).expect("cn");
105 + builder.build()
106 +}
107 +
108 +fn pem(cert: &X509) -> String {
109 + String::from_utf8(cert.to_pem().expect("cert pem")).expect("utf-8 cert")
110 +}
111 +
112 +fn key_pem(key: &PKey<Private>) -> String {
113 + String::from_utf8(key.private_key_to_pem_pkcs8().expect("key pem")).expect("utf-8 key")
114 +}
115 +
116 +/// A self-signed CA used to sign the server/client leaf certs.
117 +fn build_ca() -> (X509, PKey<Private>) {
118 + let key = gen_key();
119 + let name = x509_name("vLLM Test CA");
120 + let mut builder = X509::builder().expect("x509 builder");
121 + builder.set_version(2).expect("version");
122 + builder.set_serial_number(&serial()).expect("serial");
123 + builder.set_subject_name(&name).expect("subject");
124 + builder.set_issuer_name(&name).expect("issuer");
125 + builder.set_pubkey(&key).expect("pubkey");
126 + builder
127 + .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
128 + .expect("set nb");
129 + builder
130 + .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
131 + .expect("set na");
132 + builder
133 + .append_extension(BasicConstraints::new().critical().ca().build().expect("bc"))
134 + .expect("ext bc");
135 + builder
136 + .append_extension(
137 + KeyUsage::new().critical().key_cert_sign().crl_sign().build().expect("ku"),
138 + )
139 + .expect("ext ku");
140 + builder.sign(&key, MessageDigest::sha256()).expect("sign ca");
141 + (builder.build(), key)
142 +}
143 +
144 +/// A CA-signed leaf cert with optional subject-alternative names (IP or DNS).
145 +fn build_leaf(cn: &str, sans: &[&str], ca: &X509, ca_key: &PKey<Private>) -> (X509, PKey<Private>) {
146 + let key = gen_key();
147 + let mut builder = X509::builder().expect("x509 builder");
148 + builder.set_version(2).expect("version");
149 + builder.set_serial_number(&serial()).expect("serial");
150 + builder.set_subject_name(&x509_name(cn)).expect("subject");
151 + builder.set_issuer_name(ca.subject_name()).expect("issuer");
152 + builder.set_pubkey(&key).expect("pubkey");
153 + builder
154 + .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
155 + .expect("set nb");
156 + builder
157 + .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
158 + .expect("set na");
159 + builder
160 + .append_extension(BasicConstraints::new().build().expect("bc"))
161 + .expect("ext bc");
162 + if !sans.is_empty() {
163 + let mut san = SubjectAlternativeName::new();
164 + for entry in sans {
165 + if entry.parse::<std::net::IpAddr>().is_ok() {
166 + san.ip(entry);
167 + } else {
168 + san.dns(entry);
169 + }
170 + }
171 + let ext = san.build(&builder.x509v3_context(Some(ca), None)).expect("san");
172 + builder.append_extension(ext).expect("ext san");
173 + }
174 + builder.sign(ca_key, MessageDigest::sha256()).expect("sign leaf");
175 + (builder.build(), key)
176 +}
177 +
178 +/// A self-signed leaf not chained to the CA, for the untrusted-client test.
179 +fn build_self_signed(cn: &str) -> (X509, PKey<Private>) {
180 + let key = gen_key();
181 + let name = x509_name(cn);
182 + let mut builder = X509::builder().expect("x509 builder");
183 + builder.set_version(2).expect("version");
184 + builder.set_serial_number(&serial()).expect("serial");
185 + builder.set_subject_name(&name).expect("subject");
186 + builder.set_issuer_name(&name).expect("issuer");
187 + builder.set_pubkey(&key).expect("pubkey");
188 + builder
189 + .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
190 + .expect("set nb");
191 + builder
192 + .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
193 + .expect("set na");
194 + builder
195 + .append_extension(BasicConstraints::new().build().expect("bc"))
196 + .expect("ext bc");
197 + builder.sign(&key, MessageDigest::sha256()).expect("sign self");
198 + (builder.build(), key)
199 +}
200 +
201 +/// A CA-capable intermediate signed by the root, for the full-chain test.
202 +fn build_intermediate(ca: &X509, ca_key: &PKey<Private>) -> (X509, PKey<Private>) {
203 + let key = gen_key();
204 + let mut builder = X509::builder().expect("x509 builder");
205 + builder.set_version(2).expect("version");
206 + builder.set_serial_number(&serial()).expect("serial");
207 + builder
208 + .set_subject_name(&x509_name("vLLM Test Intermediate CA"))
209 + .expect("subject");
210 + builder.set_issuer_name(ca.subject_name()).expect("issuer");
211 + builder.set_pubkey(&key).expect("pubkey");
212 + builder
213 + .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
214 + .expect("set nb");
215 + builder
216 + .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
217 + .expect("set na");
218 + builder
219 + .append_extension(BasicConstraints::new().critical().ca().build().expect("bc"))
220 + .expect("ext bc");
221 + builder
222 + .append_extension(
223 + KeyUsage::new().critical().key_cert_sign().crl_sign().build().expect("ku"),
224 + )
225 + .expect("ext ku");
226 + builder.sign(ca_key, MessageDigest::sha256()).expect("sign intermediate");
227 + (builder.build(), key)
228 +}
229 +
230 +pub(crate) fn server_tls(certs: &TestCerts, cert_reqs: i32) -> TlsConfig {
231 + TlsConfig {
232 + cert_file: Some(certs.path("server.pem")),
233 + key_file: Some(certs.path("server.key")),
234 + ca_certs: (cert_reqs != 0).then(|| certs.path("ca.pem")),
235 + cert_reqs,
236 + ciphers: None,
237 + }
238 +}
239 +
240 +/// A plaintext-listener TLS config for `build_server_config` checks (`cert_reqs`
241 +/// 0, no client auth), with the cert/key files chosen by the caller.
242 +fn build_tls(certs: &TestCerts, cert: &str, key: Option<&str>) -> TlsConfig {
243 + TlsConfig {
244 + cert_file: Some(certs.path(cert)),
245 + key_file: key.map(|k| certs.path(k)),
246 + ca_certs: None,
247 + cert_reqs: 0,
248 + ciphers: None,
249 + }
250 +}
251 +
252 +/// Generous per-connection timeouts that never fire during the fast tests.
253 +const TEST_TIMEOUTS: ConnectionTimeouts = ConnectionTimeouts {
254 + handshake: Duration::from_secs(60),
255 + header_read: Duration::from_secs(5),
256 + keep_alive_enabled: true,
257 +};
258 +
259 +async fn spawn_server(tls_config: Option<TlsConfig>) -> (String, CancellationToken) {
260 + spawn_server_with_timeouts(tls_config, TEST_TIMEOUTS).await
261 +}
262 +
263 +/// Bind an ephemeral listener and serve a trivial router via the production
264 +/// `serve_listener`, optionally with TLS. The listener is bound (and thus
265 +/// accepting into the backlog) before returning, so a client may connect
266 +/// immediately without a sleep.
267 +async fn spawn_server_with_timeouts(
268 + tls_config: Option<TlsConfig>,
269 + timeouts: ConnectionTimeouts,
270 +) -> (String, CancellationToken) {
271 + let listener = Listener::bind(&HttpListenerMode::BindTcp {
272 + host: "127.0.0.1".to_string(),
273 + port: 0,
274 + })
275 + .await
276 + .expect("bind listener");
277 + let addr = listener.local_addr().expect("local addr");
278 +
279 + let server_config =
280 + tls_config.map(|cfg| tls::build_server_config(&cfg).expect("build server config"));
281 + let app = Router::new().route("/health", get(|| async { "ok" }));
282 + let shutdown = CancellationToken::new();
283 + let server_shutdown = shutdown.clone();
284 + tokio::spawn(async move {
285 + let _ = serve_listener(
286 + listener,
287 + server_config,
288 + app,
289 + server_shutdown.cancelled_owned(),
290 + timeouts,
291 + )
292 + .await;
293 + });
294 + (addr, shutdown)
295 +}
296 +
297 +/// Open a TLS connection trusting the test CA and finish the handshake,
298 +/// optionally presenting a client identity (`<name>.pem` + `<name>.key`) for
299 +/// mTLS. Hostname verification is disabled (the IP-SAN match is not under test);
300 +/// chain verification stays on, so an untrusted server cert is still rejected.
301 +async fn connect_tls(
302 + certs: &TestCerts,
303 + addr: &str,
304 + identity: Option<&str>,
305 +) -> std::io::Result<Pin<Box<SslStream<TcpStream>>>> {
306 + let tcp = TcpStream::connect(addr).await?;
307 +
308 + let mut builder = SslConnector::builder(SslMethod::tls_client()).expect("connector builder");
309 + builder.set_ca_file(certs.path("ca.pem")).expect("trust ca");
310 + if let Some(name) = identity {
311 + builder
312 + .set_certificate_chain_file(certs.path(&format!("{name}.pem")))
313 + .expect("client cert");
314 + builder
315 + .set_private_key_file(certs.path(&format!("{name}.key")), SslFiletype::PEM)
316 + .expect("client key");
317 + }
318 + let connector = builder.build();
319 + let mut config = connector.configure().expect("configure");
320 + config.set_verify_hostname(false);
321 + let ssl = config.into_ssl("127.0.0.1").expect("ssl");
322 +
323 + let mut stream = Box::pin(SslStream::new(ssl, tcp).expect("client ssl stream"));
324 + stream.as_mut().connect().await.map_err(std::io::Error::other)?;
325 + Ok(stream)
326 +}
327 +
328 +/// Issue an HTTPS GET (with `Connection: close`), optionally with an mTLS identity.
329 +async fn https_get(
330 + certs: &TestCerts,
331 + addr: &str,
332 + identity: Option<&str>,
333 +) -> std::io::Result<String> {
334 + let mut stream = connect_tls(certs, addr, identity).await?;
335 + stream
336 + .write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n")
337 + .await?;
338 + let mut response = String::new();
339 + stream.read_to_string(&mut response).await?;
340 + Ok(response)
341 +}
342 +
343 +/// Attempt a handshake offering only a legacy CBC+SHA1 suite over TLS 1.2,
344 +/// capping the version so TLS 1.3 cannot rescue the negotiation.
345 +async fn legacy_suite_handshake(certs: &TestCerts, addr: &str) -> std::io::Result<()> {
346 + let tcp = TcpStream::connect(addr).await?;
347 +
348 + let mut builder = SslConnector::builder(SslMethod::tls_client()).expect("connector builder");
349 + builder.set_ca_file(certs.path("ca.pem")).expect("trust ca");
350 + builder.set_max_proto_version(Some(SslVersion::TLS1_2)).expect("cap tls1.2");
351 + builder
352 + .set_cipher_list("ECDHE-ECDSA-AES256-SHA:@SECLEVEL=0")
353 + .expect("legacy cipher");
354 + let connector = builder.build();
355 + let mut config = connector.configure().expect("configure");
356 + config.set_verify_hostname(false);
357 + let ssl = config.into_ssl("127.0.0.1").expect("ssl");
358 +
359 + let stream = SslStream::new(ssl, tcp).expect("client ssl stream");
360 + tokio::pin!(stream);
361 + stream.as_mut().connect().await.map_err(std::io::Error::other)
362 +}
363 +
364 +async fn plain_get(addr: &str) -> std::io::Result<String> {
365 + let mut tcp = TcpStream::connect(addr).await?;
366 + tcp.write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n")
367 + .await?;
368 + let mut response = String::new();
369 + tcp.read_to_string(&mut response).await?;
370 + Ok(response)
371 +}
372 +
373 +// ============================================================================
374 +// Tests
375 +// ============================================================================
376 +
377 +#[test]
378 +fn builds_from_combined_pem() {
379 + // Key omitted: it is read from the combined cert+key file.
380 + let certs = TestCerts::generate();
381 + assert!(tls::build_server_config(&build_tls(&certs, "server_combined.pem", None)).is_ok());
382 +}
383 +
384 +#[test]
385 +fn rejects_missing_cert_file() {
386 + let certs = TestCerts::generate();
387 + assert!(tls::build_server_config(&build_tls(&certs, "does_not_exist.pem", None)).is_err());
388 +}
389 +
390 +#[test]
391 +fn accepts_valid_cipher_list() {
392 + let certs = TestCerts::generate();
393 + let mut cfg = build_tls(&certs, "server.pem", Some("server.key"));
394 + cfg.ciphers = Some("ECDHE-ECDSA-AES256-GCM-SHA384".to_string());
395 + assert!(tls::build_server_config(&cfg).is_ok());
396 +}
397 +
398 +#[test]
399 +fn rejects_invalid_cipher_list() {
400 + let certs = TestCerts::generate();
401 + let mut cfg = build_tls(&certs, "server.pem", Some("server.key"));
402 + cfg.ciphers = Some("THIS-IS-NOT-A-CIPHER".to_string());
403 + assert!(tls::build_server_config(&cfg).is_err());
404 +}
405 +
406 +#[test]
407 +fn rejects_mismatched_cert_and_key() {
408 + // check_private_key must reject a key that does not match the certificate.
409 + let certs = TestCerts::generate();
410 + let tls = build_tls(&certs, "client.pem", Some("server.key"));
411 + assert!(tls::build_server_config(&tls).is_err());
412 +}
413 +
414 +#[tokio::test]
415 +async fn https_request_succeeds_over_tls() {
416 + let certs = TestCerts::generate();
417 + let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 0))).await;
418 + let response = https_get(&certs, &addr, None).await.expect("https request");
419 + assert!(response.starts_with("HTTP/1.1 200"), "{response}");
420 + shutdown.cancel();
421 +}
422 +
423 +#[tokio::test]
424 +async fn serves_full_certificate_chain() {
425 + // Cert file holds leaf + intermediate; a client trusting only the root can
426 + // verify only if the server sends the intermediate, guarding against a
427 + // leaf-only load.
428 + let certs = TestCerts::generate();
429 + let tls = TlsConfig {
430 + cert_file: Some(certs.path("server_chain.pem")),
431 + key_file: Some(certs.path("server_chain.key")),
432 + ca_certs: None,
433 + cert_reqs: 0,
434 + ciphers: None,
435 + };
436 + let (addr, shutdown) = spawn_server(Some(tls)).await;
437 + let response = https_get(&certs, &addr, None).await.expect("chained https request");
438 + assert!(response.starts_with("HTTP/1.1 200"), "{response}");
439 + shutdown.cancel();
440 +}
441 +
442 +#[tokio::test]
443 +async fn rejects_legacy_cipher_only_client() {
444 + let certs = TestCerts::generate();
445 + let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 0))).await;
446 + let result = legacy_suite_handshake(&certs, &addr).await;
447 + assert!(result.is_err(), "legacy-only client must be rejected");
448 + shutdown.cancel();
449 +}
450 +
451 +#[tokio::test]
452 +async fn ssl_ciphers_override_widens_past_preset() {
453 + // Counterpart to rejects_legacy_cipher_only_client: --ssl-ciphers set to that
454 + // same legacy suite lets the client through, proving the override beats the preset.
455 + let certs = TestCerts::generate();
456 + let mut tls = server_tls(&certs, 0);
457 + tls.ciphers = Some("ECDHE-ECDSA-AES256-SHA:@SECLEVEL=0".to_string());
458 + let (addr, shutdown) = spawn_server(Some(tls)).await;
459 + let result = legacy_suite_handshake(&certs, &addr).await;
460 + assert!(
461 + result.is_ok(),
462 + "override must allow the legacy suite: {result:?}"
463 + );
464 + shutdown.cancel();
465 +}
466 +
467 +#[tokio::test]
468 +async fn mtls_required_rejects_client_without_certificate() {
469 + let certs = TestCerts::generate();
470 + let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 2))).await;
471 + let result = https_get(&certs, &addr, None).await;
472 + assert!(
473 + result.is_err(),
474 + "handshake must fail without a client certificate"
475 + );
476 + shutdown.cancel();
477 +}
478 +
479 +#[tokio::test]
480 +async fn mtls_required_accepts_valid_client_certificate() {
481 + let certs = TestCerts::generate();
482 + let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 2))).await;
483 + let response = https_get(&certs, &addr, Some("client")).await.expect("mtls request");
484 + assert!(response.starts_with("HTTP/1.1 200"), "{response}");
485 + shutdown.cancel();
486 +}
487 +
488 +#[tokio::test]
489 +async fn mtls_optional_allows_anonymous_and_authenticated() {
490 + let certs = TestCerts::generate();
491 + let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 1))).await;
492 + let anonymous = https_get(&certs, &addr, None).await.expect("anonymous request");
493 + assert!(anonymous.starts_with("HTTP/1.1 200"), "{anonymous}");
494 + let authenticated =
495 + https_get(&certs, &addr, Some("client")).await.expect("authenticated request");
496 + assert!(authenticated.starts_with("HTTP/1.1 200"), "{authenticated}");
497 + shutdown.cancel();
498 +}
499 +
500 +#[tokio::test]
501 +async fn mtls_rejects_untrusted_client_certificate() {
502 + // Optional (1) still verifies a presented cert, so a self-signed cert not
503 + // chained to the CA is rejected in both modes, not just required (2).
504 + let certs = TestCerts::generate();
505 + for cert_reqs in [1, 2] {
506 + let (addr, shutdown) = spawn_server(Some(server_tls(&certs, cert_reqs))).await;
507 + let result = https_get(&certs, &addr, Some("untrusted_client")).await;
508 + assert!(
509 + result.is_err(),
510 + "cert_reqs={cert_reqs}: untrusted client cert must be rejected"
511 + );
512 + shutdown.cancel();
513 + }
514 +}
515 +
516 +#[tokio::test]
517 +async fn plain_http_serves_when_tls_is_disabled() {
518 + let (addr, shutdown) = spawn_server(None).await;
519 + let response = plain_get(&addr).await.expect("http request");
520 + assert!(response.starts_with("HTTP/1.1 200"), "{response}");
521 + shutdown.cancel();
522 +}
523 +
524 +#[tokio::test]
525 +async fn tls_handshake_timeout_drops_silent_client() {
526 + // Silent client (no ClientHello) must be dropped at the handshake deadline.
527 + let certs = TestCerts::generate();
528 + let timeouts = ConnectionTimeouts {
529 + handshake: Duration::from_millis(150),
530 + header_read: Duration::from_secs(5),
531 + keep_alive_enabled: true,
532 + };
533 + let (addr, shutdown) = spawn_server_with_timeouts(Some(server_tls(&certs, 0)), timeouts).await;
534 +
535 + let mut tcp = TcpStream::connect(&addr).await.expect("connect");
536 + let mut buf = [0u8; 1];
537 + let read = tokio::time::timeout(Duration::from_secs(5), tcp.read(&mut buf)).await;
538 + assert!(
539 + matches!(read, Ok(Ok(0)) | Ok(Err(_))),
540 + "server must drop a stalled TLS handshake (expected close, got {read:?})"
541 + );
542 + shutdown.cancel();
543 +}
544 +
545 +#[tokio::test]
546 +async fn keep_alive_timeout_closes_idle_connection() {
547 + // Idle keep-alive connection must be closed at the deadline.
548 + let timeouts = ConnectionTimeouts {
549 + handshake: Duration::from_secs(60),
550 + header_read: Duration::from_millis(150),
551 + keep_alive_enabled: true,
552 + };
553 + let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
554 +
555 + let mut tcp = TcpStream::connect(&addr).await.expect("connect");
556 + // No `Connection: close`, so it stays alive until the idle deadline.
557 + tcp.write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n")
558 + .await
559 + .expect("write request");
560 +
561 + let drained = tokio::time::timeout(Duration::from_secs(5), async {
562 + let mut buf = [0u8; 1024];
563 + loop {
564 + match tcp.read(&mut buf).await {
565 + Ok(0) => return Ok(()),
566 + Ok(_) => continue,
567 + Err(err) => return Err(err),
568 + }
569 + }
570 + })
571 + .await;
572 + assert!(
573 + matches!(drained, Ok(Ok(()))),
574 + "server must close an idle keep-alive connection (got {drained:?})"
575 + );
576 + shutdown.cancel();
577 +}
578 +
579 +#[tokio::test]
580 +async fn keep_alive_timeout_closes_idle_tls_connection() {
581 + // The keep-alive idle bound lives in serve_connections, below TLS; assert it
582 + // still fires through tls-listener's post-handshake SslStream, not just plaintext.
583 + let certs = TestCerts::generate();
584 + let timeouts = ConnectionTimeouts {
585 + handshake: Duration::from_secs(60),
586 + header_read: Duration::from_millis(150),
587 + keep_alive_enabled: true,
588 + };
589 + let (addr, shutdown) = spawn_server_with_timeouts(Some(server_tls(&certs, 0)), timeouts).await;
590 +
591 + let mut stream = connect_tls(&certs, &addr, None).await.expect("handshake");
592 + // No `Connection: close`, so the connection stays alive until the idle deadline.
593 + stream
594 + .write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n")
595 + .await
596 + .expect("write request");
597 +
598 + let closed = tokio::time::timeout(Duration::from_secs(5), async {
599 + let mut buf = [0u8; 1024];
600 + loop {
601 + // A clean close_notify (Ok(0)) or an abrupt TLS EOF both mean the
602 + // server closed; only the outer timeout (still open) is a failure.
603 + match stream.read(&mut buf).await {
604 + Ok(0) | Err(_) => break,
605 + Ok(_) => continue,
606 + }
607 + }
608 + })
609 + .await;
610 + assert!(
611 + closed.is_ok(),
612 + "server must close an idle keep-alive TLS connection at the deadline"
613 + );
614 + shutdown.cancel();
615 +}
616 +
617 +#[tokio::test]
618 +async fn idle_timeout_closes_silent_client() {
619 + // Silent client closed by the header-read timeout (http1-only arms it from byte 0).
620 + let timeouts = ConnectionTimeouts {
621 + handshake: Duration::from_secs(60),
622 + header_read: Duration::from_millis(150),
623 + keep_alive_enabled: true,
624 + };
625 + let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
626 +
627 + let mut tcp = TcpStream::connect(&addr).await.expect("connect");
628 + let mut buf = [0u8; 1];
629 + let read = tokio::time::timeout(Duration::from_secs(5), tcp.read(&mut buf)).await;
630 + assert!(
631 + matches!(read, Ok(Ok(0)) | Ok(Err(_))),
632 + "server must close a silent client (expected close, got {read:?})"
633 + );
634 + shutdown.cancel();
635 +}
636 +
637 +#[tokio::test]
638 +async fn keep_alive_zero_disables_keep_alive() {
639 + // 0 disables keep-alive (serve, then close), like uvicorn's timeout_keep_alive=0.
640 + let timeouts = ConnectionTimeouts {
641 + handshake: Duration::from_secs(60),
642 + header_read: Duration::from_secs(5),
643 + keep_alive_enabled: false,
644 + };
645 + let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
646 +
647 + let mut tcp = TcpStream::connect(&addr).await.expect("connect");
648 + tcp.write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n")
649 + .await
650 + .expect("write request");
651 +
652 + let mut response = String::new();
653 + let read =
654 + tokio::time::timeout(Duration::from_secs(5), tcp.read_to_string(&mut response)).await;
655 + assert!(
656 + read.is_ok(),
657 + "server must close after one response, not hang"
658 + );
659 + assert!(response.starts_with("HTTP/1.1 200"), "{response}");
660 + // Assert `Connection: close`, not just 200: a 0 header-read timeout would also
661 + // serve an immediate request, so 200 alone wouldn't prove keep-alive is off.
662 + assert!(
663 + response.to_ascii_lowercase().contains("connection: close"),
664 + "keep-alive must be disabled (expected Connection: close): {response}"
665 + );
666 + shutdown.cancel();
667 +}
668 +
669 +#[tokio::test]
670 +async fn disabled_keep_alive_still_closes_silent_client() {
671 + // Even with keep-alive off, the head read stays bounded, so a silent client
672 + // is dropped rather than held open.
673 + let timeouts = ConnectionTimeouts {
674 + handshake: Duration::from_secs(60),
675 + header_read: Duration::from_millis(150),
676 + keep_alive_enabled: false,
677 + };
678 + let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
679 +
680 + let mut tcp = TcpStream::connect(&addr).await.expect("connect");
681 + let mut buf = [0u8; 1];
682 + let read = tokio::time::timeout(Duration::from_secs(5), tcp.read(&mut buf)).await;
683 + assert!(
684 + matches!(read, Ok(Ok(0)) | Ok(Err(_))),
685 + "disabled keep-alive must still close a silent client (got {read:?})"
686 + );
687 + shutdown.cancel();
688 +}
vllm/v1/utils.py
+4 -2
@@ -378,9 +378,11 @@ class RustFrontendProcessManager:
378 # The Rust `frontend` subcommand parses --args-json via serde_json,
379 # which bypasses clap and therefore ignores any `#[arg(env = ...)]`
380 # declarations on SharedRuntimeArgs fields. Forward the env-driven
381 - # ready timeout explicitly so VLLM_ENGINE_READY_TIMEOUT_S behaves the
382 - # same on both Python and Rust frontends.
381 + # values explicitly so VLLM_ENGINE_READY_TIMEOUT_S and
382 + # VLLM_HTTP_TIMEOUT_KEEP_ALIVE behave the same on both Python and Rust
383 + # frontends.
384 args_dict["engine_ready_timeout_secs"] = envs.VLLM_ENGINE_READY_TIMEOUT_S
385 + args_dict["http_timeout_keep_alive"] = envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE
386 args_json = json.dumps(args_dict, sort_keys=True)
387 cmd.extend(["--args-json", args_json])
388