@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 9d2a6f52b2

[Rust Frontend] Expose gRPC port in Python vllm serve (#59659)

Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Alec Flowers <aflowers@nvidia.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Alec committed Oct 2, 2026 at 01:14 UTC 9d2a6f52b264c20b5816c5a094b191bfbcadd74b
11 files changed +320 -64
rust/README.md
+21
@@ -43,6 +43,27 @@ For example:
43 VLLM_USE_RUST_FRONTEND=1 vllm serve Qwen/Qwen3-0.6B
44 ```
45
46 +To also expose the Rust frontend's gRPC inference and control services, add
47 +`--grpc-port`. The HTTP listener remains on `--port`:
48 +
49 +```bash
50 +VLLM_USE_RUST_FRONTEND=1 vllm serve Qwen/Qwen3-0.6B \
51 + --host 127.0.0.1 --port 8000 --grpc-port 50051
52 +```
53 +
54 +The gRPC listener binds on `--host` like HTTP, or on `127.0.0.1` when HTTP uses
55 +`--uds`. When HTTP uses TCP, its port and the gRPC port must differ unless
56 +requesting an automatically assigned port with `0`.
57 +
58 +This also works with Python-supervised hybrid data parallelism. It requires a
59 +frontend process and cannot be combined with `--headless` or
60 +`--data-parallel-multi-port-external-lb`.
61 +
62 +The separate `vllm serve MODEL --grpc --port 50051` command launches the Python
63 +SMG `VllmEngine` gRPC server in place of HTTP. The Rust listener exposes the
64 +`vllm.Inference` and `vllm.Control` services alongside HTTP. These are different
65 +gRPC APIs; `--grpc` and `--grpc-port` are mutually exclusive.
66 +
67 ### RL weight synchronization
68
69 With `VLLM_SERVER_DEV_MODE=1`, the Rust frontend supports the HTTP weight-transfer
rust/src/cmd/src/cli.rs
+24 -7
@@ -268,11 +268,6 @@ pub struct SharedRuntimeArgs {
268 #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
269 #[serde(default)]
270 pub max_logprobs: Option<i32>,
271 - /// TCP port for the gRPC Inference service. When not set, no gRPC server is
272 - /// started.
273 - #[arg(long)]
274 - #[serde(default)]
275 - pub grpc_port: Option<u16>,
271 /// Maximum time to wait for active requests to drain during shutdown.
272 #[arg(long, default_value_t = 0)]
273 #[serde(default)]
@@ -505,6 +500,7 @@ impl SharedRuntimeArgs {
500 fn into_bootstrapped_config(
501 self,
502 listen_fd: i32,
503 + grpc_listen_fd: Option<i32>,
504 input_address: String,
505 output_address: String,
506 coordinator_address: Option<String>,
@@ -539,6 +535,7 @@ impl SharedRuntimeArgs {
535 generation_config: self.generation_config,
536 served_model_name: self.served_model_name,
537 listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
538 + grpc_listener_mode: grpc_listen_fd.map(|fd| HttpListenerMode::InheritedFd { fd }),
539 tool_call_parser: self.tool_call_parser,
540 reasoning_parser: self.reasoning_parser,
541 tool_strict_level: self.tool_strict_level,
@@ -555,7 +552,6 @@ impl SharedRuntimeArgs {
552 tls,
553 api_keys: self.api_key,
554 disable_log_stats: self.disable_log_stats,
558 - grpc_port: self.grpc_port,
555 shutdown_timeout,
556 // The engine is launched and supervised by another process.
557 manages_engine: false,
@@ -569,6 +565,7 @@ impl SharedRuntimeArgs {
565 fn into_managed_config(
566 self,
567 listener_mode: HttpListenerMode,
568 + grpc_listener_mode: Option<HttpListenerMode>,
569 handshake_address: String,
570 advertised_host: String,
571 engine_count: usize,
@@ -600,6 +597,7 @@ impl SharedRuntimeArgs {
597 generation_config: self.generation_config,
598 served_model_name: self.served_model_name,
599 listener_mode,
600 + grpc_listener_mode,
601 tool_call_parser: self.tool_call_parser,
602 reasoning_parser: self.reasoning_parser,
603 tool_strict_level: self.tool_strict_level,
@@ -616,7 +614,6 @@ impl SharedRuntimeArgs {
614 tls,
615 api_keys: self.api_key,
616 disable_log_stats: self.disable_log_stats,
619 - grpc_port: self.grpc_port,
617 shutdown_timeout,
618 manages_engine,
619 keep_alive_timeout,
@@ -696,6 +693,10 @@ pub struct FrontendArgs {
693 /// supervisor.
694 #[arg(long)]
695 pub listen_fd: i32,
696 + /// Inherited gRPC listening socket file descriptor passed by the Python
697 + /// supervisor. When not set, no gRPC server is started.
698 + #[arg(long)]
699 + pub grpc_listen_fd: Option<i32>,
700 /// Frontend input ROUTER socket address that the Python engines will
701 /// connect to.
702 #[arg(long)]
@@ -731,6 +732,7 @@ impl FrontendArgs {
732 let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count);
733 self.runtime.into_bootstrapped_config(
734 self.listen_fd,
735 + self.grpc_listen_fd,
736 self.input_address,
737 self.output_address,
738 self.coordinator_address,
@@ -760,6 +762,10 @@ pub struct ServeArgs {
762 /// Unix domain socket path. If set, host and port arguments are ignored.
763 #[arg(long)]
764 pub uds: Option<String>,
765 + /// TCP port for the gRPC Inference and Control services. When not set, no
766 + /// gRPC server is started.
767 + #[arg(long)]
768 + pub grpc_port: Option<u16>,
769
770 /// Flag to print debug information about CLI argument parsing and exit.
771 #[educe(Debug(ignore))]
@@ -789,9 +795,20 @@ impl ServeArgs {
795 port: self.port,
796 },
797 };
798 + // gRPC follows the HTTP TCP host. With a Unix socket it defaults to IPv4
799 + // loopback rather than all interfaces, so the side-car is never
800 + // accidentally network-exposed.
801 + let grpc_listener_mode = self.grpc_port.map(|port| HttpListenerMode::BindTcp {
802 + host: match self.uds {
803 + Some(_) => "127.0.0.1".to_string(),
804 + None => self.host.clone(),
805 + },
806 + port,
807 + });
808
809 self.runtime.clone().into_managed_config(
810 listener_mode,
811 + grpc_listener_mode,
812 handshake_address,
813 self.managed_engine.handshake_host.clone(),
814 self.managed_engine.data_parallel_size,
rust/src/cmd/src/cli/tests.rs
+71 -6
@@ -174,6 +174,7 @@ fn serve_args_forward_python_flags_with_separator() {
174 host: "127.0.0.1",
175 port: 8000,
176 uds: None,
177 + grpc_port: None,
178 runtime: SharedRuntimeArgs {
179 model: "Qwen/Qwen3-0.6B",
180 revision: None,
@@ -188,7 +189,6 @@ fn serve_args_forward_python_flags_with_separator() {
189 renderer: Auto,
190 language_model_only: false,
191 max_logprobs: None,
191 - grpc_port: None,
192 shutdown_timeout: 0,
193 http_timeout_keep_alive: None,
194 chat_template: None,
@@ -1007,6 +1007,7 @@ fn frontend_args_accept_json() {
1007 command: Frontend(
1008 FrontendArgs {
1009 listen_fd: 3,
1010 + grpc_listen_fd: None,
1011 input_address: "ipc:///tmp/input.sock",
1012 output_address: "ipc:///tmp/output.sock",
1013 coordinator_address: Some(
@@ -1029,7 +1030,6 @@ fn frontend_args_accept_json() {
1030 renderer: Auto,
1031 language_model_only: false,
1032 max_logprobs: None,
1032 - grpc_port: None,
1033 shutdown_timeout: 0,
1034 http_timeout_keep_alive: None,
1035 chat_template: None,
@@ -1687,6 +1687,7 @@ fn serve_args_accept_handshake_aliases() {
1687 host: "127.0.0.1",
1688 port: 8000,
1689 uds: None,
1690 + grpc_port: None,
1691 runtime: SharedRuntimeArgs {
1692 model: "Qwen/Qwen3-0.6B",
1693 revision: None,
@@ -1701,7 +1702,6 @@ fn serve_args_accept_handshake_aliases() {
1702 renderer: Auto,
1703 language_model_only: false,
1704 max_logprobs: None,
1704 - grpc_port: None,
1705 shutdown_timeout: 0,
1706 http_timeout_keep_alive: None,
1707 chat_template: None,
@@ -1854,6 +1854,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1854 host: "127.0.0.1",
1855 port: 8000,
1856 },
1857 + grpc_listener_mode: None,
1858 tool_call_parser: Auto,
1859 reasoning_parser: Auto,
1860 tool_strict_level: Auto,
@@ -1887,7 +1888,6 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1888 tls: None,
1889 api_keys: [],
1890 disable_log_stats: false,
1890 - grpc_port: None,
1891 shutdown_timeout: 0ns,
1892 manages_engine: true,
1893 keep_alive_timeout: 5s,
@@ -1949,6 +1949,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1949 host: "127.0.0.1",
1950 port: 8000,
1951 },
1952 + grpc_listener_mode: None,
1953 tool_call_parser: Auto,
1954 reasoning_parser: Auto,
1955 tool_strict_level: Auto,
@@ -1982,7 +1983,6 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1983 tls: None,
1984 api_keys: [],
1985 disable_log_stats: false,
1985 - grpc_port: None,
1986 shutdown_timeout: 0ns,
1987 manages_engine: true,
1988 keep_alive_timeout: 5s,
@@ -2088,6 +2088,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
2088 listener_mode: InheritedFd {
2089 fd: 3,
2090 },
2091 + grpc_listener_mode: None,
2092 tool_call_parser: None,
2093 reasoning_parser: None,
2094 tool_strict_level: Auto,
@@ -2121,7 +2122,6 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
2122 tls: None,
2123 api_keys: [],
2124 disable_log_stats: false,
2124 - grpc_port: None,
2125 shutdown_timeout: 0ns,
2126 manages_engine: false,
2127 keep_alive_timeout: 5s,
@@ -2139,6 +2139,8 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
2139 "Qwen/Qwen3-0.6B",
2140 "--uds",
2141 "/tmp/vllm.sock",
2142 + "--grpc-port",
2143 + "50051",
2144 ])
2145 .unwrap();
2146
@@ -2153,6 +2155,69 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
2155 path: "/tmp/vllm.sock".to_string(),
2156 }
2157 );
2158 + assert_eq!(
2159 + config.grpc_listener_mode,
2160 + Some(HttpListenerMode::BindTcp {
2161 + host: "127.0.0.1".to_string(),
2162 + port: 50051,
2163 + })
2164 + );
2165 +}
2166 +
2167 +#[test]
2168 +fn serve_frontend_config_binds_grpc_on_http_host() {
2169 + let cli = Cli::try_parse_from([
2170 + "vllm-rs",
2171 + "serve",
2172 + "Qwen/Qwen3-0.6B",
2173 + "--host",
2174 + "0.0.0.0",
2175 + "--grpc-port",
2176 + "50051",
2177 + ])
2178 + .unwrap();
2179 +
2180 + let Command::Serve(args) = cli.command else {
2181 + panic!("expected serve args");
2182 + };
2183 + let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
2184 +
2185 + assert_eq!(
2186 + config.grpc_listener_mode,
2187 + Some(HttpListenerMode::BindTcp {
2188 + host: "0.0.0.0".to_string(),
2189 + port: 50051,
2190 + })
2191 + );
2192 +}
2193 +
2194 +#[test]
2195 +fn frontend_config_adopts_inherited_grpc_listener() {
2196 + let cli = Cli::try_parse_from([
2197 + "vllm-rs",
2198 + "frontend",
2199 + "--listen-fd",
2200 + "3",
2201 + "--grpc-listen-fd",
2202 + "4",
2203 + "--input-address",
2204 + "ipc:///tmp/input.sock",
2205 + "--output-address",
2206 + "ipc:///tmp/output.sock",
2207 + "--args-json",
2208 + r#"{"model_tag":"Qwen/Qwen3-0.6B","grpc_port":50051}"#,
2209 + ])
2210 + .unwrap();
2211 +
2212 + let Command::Frontend(args) = cli.command else {
2213 + panic!("expected frontend args");
2214 + };
2215 + let config = args.into_config();
2216 +
2217 + assert_eq!(
2218 + config.grpc_listener_mode,
2219 + Some(HttpListenerMode::InheritedFd { fd: 4 })
2220 + );
2221 }
2222
2223 #[test]
rust/src/server/examples/external_engine_openai_qwen.rs
+1 -1
@@ -69,6 +69,7 @@ async fn main() -> Result<()> {
69 host: "127.0.0.1".to_string(),
70 port,
71 },
72 + grpc_listener_mode: None,
73 tool_call_parser: ParserSelection::Auto,
74 reasoning_parser: ParserSelection::Auto,
75 tool_strict_level: ToolStrictLevel::Auto,
@@ -85,7 +86,6 @@ async fn main() -> Result<()> {
86 tls: None,
87 api_keys: Vec::new(),
88 disable_log_stats: false,
88 - grpc_port: None,
89 shutdown_timeout: Duration::ZERO,
90 manages_engine: false,
91 keep_alive_timeout: Duration::from_secs(5),
rust/src/server/src/config.rs
+3 -4
@@ -23,7 +23,7 @@ use vllm_text::backend::hf::HfOverrides;
23 /// when keep-alive is disabled (`0`).
24 pub const DEFAULT_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5);
25
26 -/// How the HTTP server obtains its listening socket.
26 +/// How the HTTP or gRPC server obtains its listening socket.
27 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
28 pub enum HttpListenerMode {
29 /// Bind a fresh TCP listener on the given host/port.
@@ -229,6 +229,8 @@ pub struct Config {
229 pub served_model_name: Vec<String>,
230 /// HTTP listener setup.
231 pub listener_mode: HttpListenerMode,
232 + /// gRPC listener setup. When `None`, no gRPC server is started.
233 + pub grpc_listener_mode: Option<HttpListenerMode>,
234 /// Tool-call parser selection.
235 pub tool_call_parser: ParserSelection,
236 /// Reasoning parser selection.
@@ -272,9 +274,6 @@ pub struct Config {
274 /// engine-reported scheduler stats and request lifecycle events are not
275 /// exported.
276 pub disable_log_stats: bool,
275 - /// TCP port for the gRPC Inference service. When `None`, no gRPC server is
276 - /// started.
277 - pub grpc_port: Option<u16>,
277 /// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
278 pub shutdown_timeout: Duration,
279 /// Whether the caller manages the engine process and shuts it down when
rust/src/server/src/lib.rs
+4 -37
@@ -37,7 +37,6 @@ use hyper_util::rt::{TokioIo, TokioTimer};
37 use hyper_util::server::graceful::GracefulShutdown;
38 use hyper_util::service::TowerToHyperService;
39 pub use render::{RenderConfig, serve_render};
40 -use tokio::net::TcpListener;
40 use tokio::time::{Instant, sleep_until};
41 use tokio_util::sync::CancellationToken;
42 use tonic::transport::Server as TonicServer;
@@ -74,17 +73,6 @@ fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Ve
73 }
74 }
75
77 -/// Choose the gRPC listener host. It follows the HTTP TCP host when there is
78 -/// one; otherwise (unix socket or inherited fd) it defaults to IPv4 loopback
79 -/// rather than all interfaces, so the side-car is never accidentally
80 -/// network-exposed.
81 -fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
82 - match listener_mode {
83 - HttpListenerMode::BindTcp { host, .. } => host.as_str(),
84 - HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "127.0.0.1",
85 - }
86 -}
87 -
76 /// Build the shared application state for one configured model and one engine
77 /// client, including any static LoRA adapters from `--lora-modules`.
78 async fn build_state(config: &Config) -> Result<Arc<AppState>> {
@@ -229,13 +217,11 @@ where
217 // Optionally bind the gRPC Inference server on a separate port. Bind
218 // synchronously here so bind errors (port in use, permission denied, ...)
219 // surface before serving rather than being deferred until shutdown.
232 - let grpc_setup = if let Some(grpc_port) = config.grpc_port {
233 - let grpc_host = grpc_bind_host(&config.listener_mode);
234 - let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
220 + let grpc_setup = if let Some(grpc_listener_mode) = &config.grpc_listener_mode {
221 + let grpc_listener = Listener::bind(grpc_listener_mode)
222 .await
236 - .with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
237 - let addr = grpc_listener.local_addr()?;
238 - let grpc_listener = Listener::Tcp(grpc_listener);
223 + .with_context(|| format!("failed to bind gRPC listener for {grpc_listener_mode:?}"))?;
224 + let addr = grpc_listener.local_addr_display()?;
225 // gRPC reuses the HTTP TLS config (same SslContext) plus ALPN h2.
226 let grpc_tls = config
227 .tls
@@ -479,23 +465,4 @@ mod tests {
465 served_names
466 );
467 }
482 -
483 - #[test]
484 - fn grpc_bind_host_follows_http_tcp_host() {
485 - let mode = HttpListenerMode::BindTcp {
486 - host: "0.0.0.0".to_string(),
487 - port: 8000,
488 - };
489 - assert_eq!(grpc_bind_host(&mode), "0.0.0.0");
490 - }
491 -
492 - #[test]
493 - fn grpc_bind_host_defaults_to_loopback_without_tcp_host() {
494 - let unix = HttpListenerMode::BindUnix {
495 - path: "/tmp/vllm.sock".to_string(),
496 - };
497 - let inherited = HttpListenerMode::InheritedFd { fd: 3 };
498 - assert_eq!(grpc_bind_host(&unix), "127.0.0.1");
499 - assert_eq!(grpc_bind_host(&inherited), "127.0.0.1");
500 - }
468 }
tests/entrypoints/launchers/api_server/test_api_server_process_manager.py
+53
@@ -536,3 +536,56 @@ def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_p
536 sock.close()
537
538 assert '"model_tag": "org/model"' in caplog.text
539 +
540 +
541 +def test_rust_frontend_inherits_grpc_listener(monkeypatch):
542 + """The Python-bound gRPC socket is passed to Rust like the HTTP one."""
543 + import subprocess as subprocess_mod
544 +
545 + from vllm.entrypoints.launchers.cli_args import make_arg_parser
546 + from vllm.utils.argparse_utils import FlexibleArgumentParser
547 + from vllm.v1.utils import RustFrontendProcessManager
548 +
549 + args = make_arg_parser(FlexibleArgumentParser()).parse_args(
550 + ["--model", "org/model", "--grpc-port", "50051"]
551 + )
552 +
553 + class _FakeProc:
554 + pid = 4321
555 + returncode = 0
556 +
557 + def poll(self):
558 + return 0
559 +
560 + def wait(self, timeout=None):
561 + return 0
562 +
563 + popen_calls = []
564 +
565 + def fake_popen(cmd, **kwargs):
566 + popen_calls.append((cmd, kwargs))
567 + return _FakeProc()
568 +
569 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
570 + grpc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
571 + try:
572 + monkeypatch.setattr(subprocess_mod, "Popen", fake_popen)
573 + RustFrontendProcessManager(
574 + binary_path="/nonexistent/vllm-rs",
575 + sock=sock,
576 + args=args,
577 + input_address="ipc:///tmp/in",
578 + output_address="ipc:///tmp/out",
579 + engine_start_index=0,
580 + engine_count=1,
581 + data_parallel_size=1,
582 + grpc_sock=grpc_sock,
583 + )
584 + [(cmd, kwargs)] = popen_calls
585 + grpc_fd = grpc_sock.fileno()
586 + flag_index = cmd.index("--grpc-listen-fd")
587 + assert cmd[flag_index + 1] == str(grpc_fd)
588 + assert list(kwargs["pass_fds"]) == [sock.fileno(), grpc_fd]
589 + finally:
590 + sock.close()
591 + grpc_sock.close()
tests/entrypoints/launchers/test_cli_args.py
+83 -6
@@ -28,7 +28,7 @@ assert CHATML_JINJA_PATH.exists()
28
29 def _build_vllm_parsers():
30 vllm_parser = FlexibleArgumentParser()
31 - subparsers = vllm_parser.add_subparsers()
31 + subparsers = vllm_parser.add_subparsers(dest="subparser")
32 serve_parser = subparsers.add_parser("serve")
33 make_arg_parser(serve_parser)
34 return {"vllm": vllm_parser, "vllm serve": serve_parser}
@@ -173,6 +173,75 @@ def test_multiple_valid_inputs(serve_parser):
173
174
175 ### Tests for serve argument validation that run prior to loading
176 +@pytest.mark.parametrize(
177 + "port, extra_args",
178 + [
179 + (50051, []),
180 + (0, ["--port", "0"]),
181 + (50051, ["--port", "50051", "--uds", "/tmp/vllm.sock"]),
182 + ],
183 +)
184 +def test_rust_grpc_port_accepts_frontend_launch(
185 + vllm_parser, monkeypatch, port, extra_args
186 +):
187 + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
188 + args = vllm_parser.parse_args(
189 + [
190 + "serve",
191 + "--grpc-port",
192 + str(port),
193 + "--data-parallel-size",
194 + "8",
195 + "--data-parallel-size-local",
196 + "4",
197 + "--data-parallel-start-rank",
198 + "4",
199 + "--data-parallel-hybrid-lb",
200 + *extra_args,
201 + ]
202 + )
203 + validate_parsed_serve_args(args)
204 + assert args.grpc_port == port
205 +
206 +
207 +@pytest.mark.parametrize(
208 + "rust_enabled, extra_args, port",
209 + [
210 + (False, [], 50051),
211 + (False, ["--grpc"], 50051),
212 + (True, ["--grpc"], 50051),
213 + (True, ["--headless"], 50051),
214 + (True, ["--api-server-count", "0"], 50051),
215 + (True, ["--api-server-count", "-1"], 50051),
216 + (True, ["--data-parallel-multi-port-external-lb"], 50051),
217 + (True, ["--port", "50051"], 50051),
218 + (True, [], -1),
219 + (True, [], 65536),
220 + ],
221 +)
222 +def test_rust_grpc_port_rejects_incompatible_launch(
223 + vllm_parser, monkeypatch, rust_enabled, extra_args, port
224 +):
225 + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1" if rust_enabled else "0")
226 + args = vllm_parser.parse_args(["serve", "--grpc-port", str(port), *extra_args])
227 + error = (
228 + "--grpc and --grpc-port are mutually exclusive"
229 + if "--grpc" in extra_args
230 + else "--grpc-port"
231 + )
232 + with pytest.raises(ValueError, match=error):
233 + validate_parsed_serve_args(args)
234 +
235 +
236 +def test_rust_grpc_port_requires_serve_subcommand(monkeypatch):
237 + """Direct Python render/API entrypoints must reject a Rust-only listener."""
238 + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
239 + parser = make_arg_parser(FlexibleArgumentParser())
240 + args = parser.parse_args(["--grpc-port", "50051"])
241 + with pytest.raises(ValueError, match="--grpc-port requires"):
242 + validate_parsed_serve_args(args)
243 +
244 +
245 def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser):
246 """Ensure validation fails if tool choice is enabled with no call parser."""
247 # If we enable-auto-tool-choice, explode with no tool-call-parser
@@ -265,13 +334,21 @@ def launch_render_parser():
334 return _build_launch_render_parser()
335
336
268 -def test_launch_render_validates_serve_args(launch_render_parser):
337 +@pytest.mark.parametrize(
338 + "cli_args, error",
339 + [
340 + (["--enable-auto-tool-choice"], TypeError),
341 + (["--grpc-port", "50051"], ValueError),
342 + ],
343 +)
344 +def test_launch_render_validates_serve_args(
345 + launch_render_parser, monkeypatch, cli_args, error
346 +):
347 """`vllm launch render` reuses the serve parser, so it gets the serve checks"""
270 - args = launch_render_parser.parse_args(
271 - args=["launch", "render", "--enable-auto-tool-choice"]
272 - )
348 + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1")
349 + args = launch_render_parser.parse_args(args=["launch", "render", *cli_args])
350 assert args.subparser == "launch"
274 - with pytest.raises(TypeError):
351 + with pytest.raises(error):
352 validate_parsed_serve_args(args)
353
354
vllm/entrypoints/cli/serve.py
+9
@@ -17,6 +17,7 @@ from vllm.entrypoints.launchers.cli_args import (
17 validate_parsed_serve_args,
18 )
19 from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
20 +from vllm.entrypoints.launchers.launcher import create_server_socket
21 from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
22 from vllm.logger import init_logger
23 from vllm.reasoning import ReasoningParserManager
@@ -295,6 +296,13 @@ def run_multi_api_server(args: argparse.Namespace):
296 signal.signal(signal.SIGINT, signal_handler)
297
298 listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
299 + # `--grpc-port` is only accepted for the Rust frontend, which inherits this
300 + # listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4
301 + # loopback when HTTP uses a Unix socket.
302 + grpc_sock = None
303 + if args.grpc_port is not None:
304 + grpc_host = "127.0.0.1" if args.uds else (args.host or "")
305 + grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False)
306
307 engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
308 propagate_flash_late_interaction(args, engine_args)
@@ -355,6 +363,7 @@ def run_multi_api_server(args: argparse.Namespace):
363 api_server_manager = RustFrontendProcessManager(
364 binary_path=rust_frontend_path,
365 sock=sock,
366 + grpc_sock=grpc_sock,
367 args=args,
368 input_address=addresses.inputs[0],
369 output_address=addresses.outputs[0],
vllm/entrypoints/launchers/cli_args.py
+43 -2
@@ -284,6 +284,11 @@ class FrontendArgs(BaseFrontendArgs):
284 """Host name."""
285 port: int = 8000
286 """Port number."""
287 + grpc_port: int | None = None
288 + """Enable the Rust frontend's additional gRPC Inference and Control services
289 + on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on
290 + `--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined
291 + with the Python gRPC server's `--grpc` flag."""
292 data_parallel_supervisor_port: int = 9256
293 """HTTP port for aggregated health endpoints in multi-port external LB
294 mode."""
@@ -451,8 +456,10 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
456 "--grpc",
457 action="store_true",
458 default=False,
454 - help="Launch a gRPC server instead of the HTTP OpenAI-compatible "
455 - "server. Requires: pip install vllm[grpc].",
459 + help="Launch the Python gRPC (SMG VllmEngine) server on --port instead "
460 + "of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. "
461 + "Cannot be combined with --grpc-port, which enables the Rust "
462 + "frontend's gRPC services.",
463 )
464 parser = FrontendArgs.add_cli_args(parser)
465 parser = AsyncEngineArgs.add_cli_args(parser)
@@ -460,6 +467,38 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
467 return parser
468
469
470 +def validate_grpc_port_arg(args: argparse.Namespace) -> None:
471 + """Validate the Rust frontend's optional gRPC listener."""
472 + if getattr(args, "grpc_port", None) is None:
473 + return
474 +
475 + if args.grpc:
476 + raise ValueError(
477 + "--grpc and --grpc-port are mutually exclusive. Use --grpc "
478 + "--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 "
479 + "with --grpc-port for the Rust frontend."
480 + )
481 + if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve":
482 + raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve")
483 + if args.headless or (
484 + args.api_server_count is not None and args.api_server_count <= 0
485 + ):
486 + raise ValueError(
487 + "--grpc-port requires a Rust frontend; remove --headless "
488 + "and non-positive --api-server-count"
489 + )
490 + if args.data_parallel_multi_port_external_lb:
491 + raise ValueError(
492 + "--grpc-port is incompatible with "
493 + "--data-parallel-multi-port-external-lb: "
494 + "its frontends would share the same gRPC port"
495 + )
496 + if not 0 <= args.grpc_port <= 65535:
497 + raise ValueError("--grpc-port must be between 0 and 65535")
498 + if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds:
499 + raise ValueError("--grpc-port must differ from --port when HTTP uses TCP")
500 +
501 +
502 def validate_parsed_serve_args(args: argparse.Namespace):
503 """Quick checks for model serve args that raise prior to loading."""
504 # `vllm launch <component>` builds its parser with make_arg_parser too (see
@@ -478,6 +517,8 @@ def validate_parsed_serve_args(args: argparse.Namespace):
517 "the checkpoint's output format."
518 )
519
520 + validate_grpc_port_arg(args)
521 +
522 # Enable auto tool needs a tool call parser to be valid
523 if args.enable_auto_tool_choice and not args.tool_call_parser:
524 raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
vllm/v1/utils.py
+8 -1
@@ -354,6 +354,7 @@ class RustFrontendProcessManager:
354 engine_count: int,
355 data_parallel_size: int,
356 stats_update_address: str | None = None,
357 + grpc_sock: Any | None = None,
358 ):
359 import os
360 import subprocess
@@ -377,6 +378,12 @@ class RustFrontendProcessManager:
378 "--data-parallel-size",
379 str(data_parallel_size),
380 ]
381 + pass_fds = [fd]
382 + if grpc_sock is not None:
383 + grpc_fd = grpc_sock.fileno()
384 + os.set_inheritable(grpc_fd, True)
385 + cmd.extend(["--grpc-listen-fd", str(grpc_fd)])
386 + pass_fds.append(grpc_fd)
387 if stats_update_address is not None:
388 cmd.extend(["--coordinator-address", stats_update_address])
389 from vllm.entrypoints.serve.utils.api_utils import jsonify_non_default_args
@@ -418,7 +425,7 @@ class RustFrontendProcessManager:
425
426 redacted_json = json.dumps(redact_sensitive_args(args_dict), sort_keys=True)
427 logger.info("Launching Rust frontend: %s", " ".join(cmd[:-1] + [redacted_json]))
421 - self._proc = subprocess.Popen(cmd, pass_fds=(fd,))
428 + self._proc = subprocess.Popen(cmd, pass_fds=pass_fds)
429
430 # Create a process wrapper with a sentinel fd for monitoring
431 self.processes: list[_SubprocessWrapper] = [