@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cca3365b73
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Rust Frontend] Add CORS support (#45753)

Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>

Tahsin Tunan committed Jun 16, 2026 at 19:47 UTC cca3365b733b5973d60a6631df8ee65ae53666a9
12 files changed +732 -42
rust/Cargo.toml
+1 -1
@@ -105,7 +105,7 @@ tonic-prost = "0.14.5"
105 tonic-prost-build = "0.14.5"
106 tool-parser = "1.2.0"
107 tower = { version = "0.5.3", features = ["util"] }
108 -tower-http = { version = "0.6.8", features = ["trace"] }
108 +tower-http = { version = "0.6.8", features = ["cors", "trace"] }
109 tracing = { version = "0.1.44", features = ["release_max_level_debug"] }
110 tracing-futures = { version = "0.2.5", features = ["futures-03"] }
111 tracing-subscriber = { version = "0.3.20", features = ["env-filter", "fmt"] }
rust/src/cmd/src/cli.rs
+50 -2
@@ -23,8 +23,8 @@ use vllm_engine_core_client::TransportMode;
23 use vllm_managed_engine::ManagedEngineConfig;
24 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
25 use vllm_server::{
26 - ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode,
27 - ParserSelection, RendererSelection,
26 + ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
27 + HttpListenerMode, ParserSelection, RendererSelection,
28 };
29
30 use crate::cli::unsupported::UnsupportedArgs;
@@ -84,6 +84,13 @@ pub enum Command {
84 Serve(ServeArgs),
85 }
86
87 +/// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for
88 +/// the CORS list arguments (e.g. `--allowed-origins '["*"]'`). Parsing the whole
89 +/// value as one item keeps clap from treating the field as a repeated flag.
90 +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
91 +#[serde(transparent)]
92 +pub struct JsonStringList(pub Vec<String>);
93 +
94 /// Runtime arguments shared by the external-engine and managed-engine paths.
95 #[serde_as]
96 #[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
@@ -220,6 +227,30 @@ pub struct SharedRuntimeArgs {
227 #[serde(default)]
228 pub served_model_name: Vec<String>,
229
230 + /// CORS allowed origins as a JSON list. `["*"]` allows any origin.
231 + #[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
232 + #[serde(default = "default_cors_wildcard")]
233 + pub allowed_origins: JsonStringList,
234 +
235 + /// CORS allowed methods as a JSON list. `["*"]` allows the standard set.
236 + #[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
237 + #[serde(default = "default_cors_wildcard")]
238 + pub allowed_methods: JsonStringList,
239 +
240 + /// CORS allowed request headers as a JSON list. `["*"]` mirrors the request.
241 + #[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
242 + #[serde(default = "default_cors_wildcard")]
243 + pub allowed_headers: JsonStringList,
244 +
245 + /// Allow CORS credentials (cookies, authorization headers).
246 + #[arg(
247 + long,
248 + default_missing_value = "true",
249 + num_args = 0..=1
250 + )]
251 + #[serde(default)]
252 + pub allow_credentials: bool,
253 +
254 /// Unsupported Python vLLM frontend arguments recognized but not yet
255 /// implemented in Rust.
256 #[educe(Debug(ignore))]
@@ -264,6 +295,7 @@ impl SharedRuntimeArgs {
295 let ready_timeout = self.ready_timeout();
296 let shutdown_timeout = self.shutdown_timeout();
297 let api_server_options = self.api_server_options();
298 + let cors = self.cors_config();
299
300 Config {
301 transport_mode: TransportMode::Bootstrapped {
@@ -288,6 +320,7 @@ impl SharedRuntimeArgs {
320 chat_template_content_format: self.chat_template_content_format,
321 max_logprobs: self.max_logprobs,
322 api_server_options,
323 + cors,
324 api_keys: self.api_key,
325 disable_log_stats: self.disable_log_stats,
326 grpc_port: self.grpc_port,
@@ -309,6 +342,7 @@ impl SharedRuntimeArgs {
342 let ready_timeout = self.ready_timeout();
343 let shutdown_timeout = self.shutdown_timeout();
344 let api_server_options = self.api_server_options();
345 + let cors = self.cors_config();
346
347 Config {
348 transport_mode: TransportMode::HandshakeOwner {
@@ -332,6 +366,7 @@ impl SharedRuntimeArgs {
366 chat_template_content_format: self.chat_template_content_format,
367 max_logprobs: self.max_logprobs,
368 api_server_options,
369 + cors,
370 api_keys: self.api_key,
371 disable_log_stats: self.disable_log_stats,
372 grpc_port: self.grpc_port,
@@ -346,12 +381,25 @@ impl SharedRuntimeArgs {
381 enable_request_id_headers: self.enable_request_id_headers,
382 }
383 }
384 +
385 + fn cors_config(&self) -> CorsConfig {
386 + CorsConfig {
387 + allow_origins: self.allowed_origins.0.clone(),
388 + allow_methods: self.allowed_methods.0.clone(),
389 + allow_headers: self.allowed_headers.0.clone(),
390 + allow_credentials: self.allow_credentials,
391 + }
392 + }
393 }
394
395 fn default_engine_ready_timeout_secs() -> u64 {
396 600
397 }
398
399 +fn default_cors_wildcard() -> JsonStringList {
400 + JsonStringList(vec!["*".to_string()])
401 +}
402 +
403 fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
404 serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
405 }
rust/src/cmd/src/cli/tests.rs
+165 -11
@@ -49,6 +49,22 @@ fn serve_args_forward_python_flags_with_separator() {
49 enable_request_id_headers: false,
50 disable_log_stats: false,
51 served_model_name: [],
52 + allowed_origins: JsonStringList(
53 + [
54 + "*",
55 + ],
56 + ),
57 + allowed_methods: JsonStringList(
58 + [
59 + "*",
60 + ],
61 + ),
62 + allowed_headers: JsonStringList(
63 + [
64 + "*",
65 + ],
66 + ),
67 + allow_credentials: false,
68 },
69 managed_engine: ManagedEngineArgs {
70 python: "../vllm/.venv/bin/python",
@@ -336,11 +352,17 @@ fn serve_args_reject_unknown_renderer_value() {
352
353 #[test]
354 fn serve_args_reject_unsupported_flag_arg() {
339 - let error = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--allow-credentials"])
340 - .unwrap_err();
355 + let error = Cli::try_parse_from([
356 + "vllm-rs",
357 + "serve",
358 + "Qwen/Qwen3-0.6B",
359 + "--ssl-keyfile",
360 + "/tmp/key.pem",
361 + ])
362 + .unwrap_err();
363
364 expect![[r#"
343 - error: invalid value 'true' for '--allow-credentials [<ALLOW_CREDENTIALS>]': argument is not implemented in Rust frontend yet
365 + error: invalid value '/tmp/key.pem' for '--ssl-keyfile <SSL_KEYFILE>': argument is not implemented in Rust frontend yet
366
367 Remove this unsupported argument to continue.
368
@@ -348,8 +370,7 @@ fn serve_args_reject_unsupported_flag_arg() {
370 This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
371
372 For more information, try '--help'.
351 - "#]]
352 - .assert_eq(&error.to_string());
373 + "#]].assert_eq(&error.to_string());
374 }
375
376 #[test]
@@ -423,6 +444,22 @@ fn frontend_args_accept_json() {
444 enable_request_id_headers: false,
445 disable_log_stats: false,
446 served_model_name: [],
447 + allowed_origins: JsonStringList(
448 + [
449 + "*",
450 + ],
451 + ),
452 + allowed_methods: JsonStringList(
453 + [
454 + "*",
455 + ],
456 + ),
457 + allowed_headers: JsonStringList(
458 + [
459 + "*",
460 + ],
461 + ),
462 + allow_credentials: false,
463 },
464 },
465 ),
@@ -558,6 +595,71 @@ fn frontend_args_json_sets_prompt_tokens_details_flag() {
595 assert!(args.runtime.enable_prompt_tokens_details);
596 }
597
598 +#[test]
599 +fn serve_args_parse_cors_flags() {
600 + let cli = Cli::try_parse_from([
601 + "vllm-rs",
602 + "serve",
603 + "Qwen/Qwen3-0.6B",
604 + "--allowed-origins",
605 + r#"["http://a.com","http://b.com"]"#,
606 + "--allowed-methods",
607 + r#"["GET","POST"]"#,
608 + "--allow-credentials",
609 + ])
610 + .unwrap();
611 +
612 + let Command::Serve(serve) = cli.command else {
613 + panic!("expected serve args");
614 + };
615 + assert_eq!(
616 + serve.runtime.allowed_origins.0,
617 + ["http://a.com", "http://b.com"]
618 + );
619 + assert_eq!(serve.runtime.allowed_methods.0, ["GET", "POST"]);
620 + assert!(serve.runtime.allow_credentials);
621 + // Unspecified lists keep the permissive default.
622 + assert_eq!(serve.runtime.allowed_headers.0, ["*"]);
623 +}
624 +
625 +#[test]
626 +fn serve_args_cors_defaults_are_permissive() {
627 + let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
628 +
629 + let Command::Serve(serve) = cli.command else {
630 + panic!("expected serve args");
631 + };
632 + assert_eq!(serve.runtime.allowed_origins.0, ["*"]);
633 + assert_eq!(serve.runtime.allowed_methods.0, ["*"]);
634 + assert_eq!(serve.runtime.allowed_headers.0, ["*"]);
635 + assert!(!serve.runtime.allow_credentials);
636 +}
637 +
638 +#[test]
639 +fn frontend_args_json_parses_cors_fields() {
640 + let cli = Cli::try_parse_from([
641 + "vllm-rs",
642 + "frontend",
643 + "--listen-fd",
644 + "3",
645 + "--input-address",
646 + "ipc:///tmp/input.sock",
647 + "--output-address",
648 + "ipc:///tmp/output.sock",
649 + "--args-json",
650 + r#"{"model_tag":"Qwen/Qwen3-0.6B","allowed_origins":["http://a.com"],"allow_credentials":true}"#,
651 + ])
652 + .unwrap();
653 +
654 + let Command::Frontend(args) = cli.command else {
655 + panic!("expected frontend args");
656 + };
657 + assert_eq!(args.runtime.allowed_origins.0, ["http://a.com"]);
658 + assert!(args.runtime.allow_credentials);
659 + // Unspecified lists fall back to the permissive default via serde.
660 + assert_eq!(args.runtime.allowed_methods.0, ["*"]);
661 +}
662 +
663 #[test]
664 fn frontend_args_json_rejects_unsupported_fields() {
665 let error = Cli::try_parse_from([
@@ -570,14 +672,14 @@ fn frontend_args_json_rejects_unsupported_fields() {
672 "--output-address",
673 "ipc:///tmp/output.sock",
674 "--args-json",
573 - r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}"#,
675 + r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}"#,
676 ])
677 .unwrap_err();
678
679 expect![[r#"
578 - error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true}' for '--args-json <JSON>':
680 + error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
681 The following arguments are not implemented in Rust frontend yet:
580 - - allow_credentials
682 + - ssl_keyfile
683
684 Remove these arguments to continue.
685
@@ -597,15 +699,15 @@ fn frontend_args_json_aggregates_multiple_unsupported_fields() {
699 "--output-address",
700 "ipc:///tmp/output.sock",
701 "--args-json",
600 - r#"{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"ssl_keyfile":"/tmp/key.pem"}"#,
702 + r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}"#,
703 ])
704 .unwrap_err();
705
706 let actual = error.to_string().replace(": \n", ":\n");
707 expect![[r#"
606 - error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","allow_credentials":true,"ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
708 + error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
709 The following arguments are not implemented in Rust frontend yet:
608 - - allow_credentials
710 + - response_role
711 - ssl_keyfile
712
713 Remove these arguments to continue.
@@ -830,6 +932,22 @@ fn serve_args_accept_handshake_aliases() {
932 enable_request_id_headers: false,
933 disable_log_stats: false,
934 served_model_name: [],
935 + allowed_origins: JsonStringList(
936 + [
937 + "*",
938 + ],
939 + ),
940 + allowed_methods: JsonStringList(
941 + [
942 + "*",
943 + ],
944 + ),
945 + allowed_headers: JsonStringList(
946 + [
947 + "*",
948 + ],
949 + ),
950 + allow_credentials: false,
951 },
952 managed_engine: ManagedEngineArgs {
953 python: "python3",
@@ -951,6 +1069,18 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1069 enable_prompt_tokens_details: false,
1070 enable_request_id_headers: false,
1071 },
1072 + cors: CorsConfig {
1073 + allow_origins: [
1074 + "*",
1075 + ],
1076 + allow_methods: [
1077 + "*",
1078 + ],
1079 + allow_headers: [
1080 + "*",
1081 + ],
1082 + allow_credentials: false,
1083 + },
1084 api_keys: [],
1085 disable_log_stats: false,
1086 grpc_port: None,
@@ -1020,6 +1150,18 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1150 enable_prompt_tokens_details: false,
1151 enable_request_id_headers: false,
1152 },
1153 + cors: CorsConfig {
1154 + allow_origins: [
1155 + "*",
1156 + ],
1157 + allow_methods: [
1158 + "*",
1159 + ],
1160 + allow_headers: [
1161 + "*",
1162 + ],
1163 + allow_credentials: false,
1164 + },
1165 api_keys: [],
1166 disable_log_stats: false,
1167 grpc_port: None,
@@ -1104,6 +1246,18 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
1246 enable_prompt_tokens_details: false,
1247 enable_request_id_headers: false,
1248 },
1249 + cors: CorsConfig {
1250 + allow_origins: [
1251 + "*",
1252 + ],
1253 + allow_methods: [
1254 + "*",
1255 + ],
1256 + allow_headers: [
1257 + "*",
1258 + ],
1259 + allow_credentials: false,
1260 + },
1261 api_keys: [],
1262 disable_log_stats: false,
1263 grpc_port: None,
rust/src/cmd/src/cli/unsupported.rs
-21
@@ -526,27 +526,6 @@ pub struct ServerUnsupportedArgs {
526 #[arg(long)]
527 pub disable_access_log_for_endpoints: Option<Noop>,
528
529 - /// Allow credentials.
530 - #[arg(
531 - long,
532 - visible_alias = "no-allow-credentials",
533 - default_missing_value = "true",
534 - num_args = 0..=1
535 - )]
536 - pub allow_credentials: Option<Unsupported>,
537 -
538 - /// Allowed origins.
539 - #[arg(long)]
540 - pub allowed_origins: Option<Unsupported>,
541 -
542 - /// Allowed methods.
543 - #[arg(long)]
544 - pub allowed_methods: Option<Unsupported>,
545 -
546 - /// Allowed headers.
547 - #[arg(long)]
548 - pub allowed_headers: Option<Unsupported>,
549 -
529 /// The file path to the SSL key file.
530 #[arg(long)]
531 pub ssl_keyfile: Option<Unsupported>,
rust/src/server/examples/external_engine_openai_qwen.rs
+3 -2
@@ -14,8 +14,8 @@ use tokio_util::sync::CancellationToken;
14 use tracing_subscriber::EnvFilter;
15 use vllm_engine_core_client::TransportMode;
16 use vllm_server::{
17 - ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode,
18 - ParserSelection, RendererSelection, serve,
17 + ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
18 + HttpListenerMode, ParserSelection, RendererSelection, serve,
19 };
20
21 #[derive(Debug, Parser)]
@@ -70,6 +70,7 @@ async fn main() -> Result<()> {
70 chat_template_content_format: ChatTemplateContentFormatOption::Auto,
71 max_logprobs: None,
72 api_server_options: ApiServerOptions::default(),
73 + cors: CorsConfig::default(),
74 api_keys: Vec::new(),
75 disable_log_stats: false,
76 grpc_port: None,
rust/src/server/src/config.rs
+57
@@ -3,6 +3,7 @@ use std::fmt;
3 use std::time::Duration;
4
5 use anyhow::{Result, bail};
6 +use axum::http::{HeaderName, HeaderValue, Method};
7 use educe::Educe;
8 use serde::Serialize;
9 use serde_json::Value;
@@ -45,6 +46,59 @@ pub struct ApiServerOptions {
46 pub enable_request_id_headers: bool,
47 }
48
49 +/// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive.
50 +#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
51 +pub struct CorsConfig {
52 + /// Allowed origins. `["*"]` allows any origin.
53 + pub allow_origins: Vec<String>,
54 + /// Allowed methods. `["*"]` allows the standard method set.
55 + pub allow_methods: Vec<String>,
56 + /// Allowed request headers. `["*"]` mirrors the requested headers.
57 + pub allow_headers: Vec<String>,
58 + /// Whether to allow credentials (cookies, authorization headers).
59 + pub allow_credentials: bool,
60 +}
61 +
62 +impl Default for CorsConfig {
63 + fn default() -> Self {
64 + Self {
65 + allow_origins: vec!["*".to_string()],
66 + allow_methods: vec!["*".to_string()],
67 + allow_headers: vec!["*".to_string()],
68 + allow_credentials: false,
69 + }
70 + }
71 +}
72 +
73 +impl CorsConfig {
74 + /// Validate that non-wildcard values parse into HTTP types, so the CORS
75 + /// layer can be built infallibly after startup validation has run.
76 + pub fn validate(&self) -> Result<()> {
77 + for origin in &self.allow_origins {
78 + if origin != "*" {
79 + origin.parse::<HeaderValue>().map_err(|e| {
80 + anyhow::anyhow!("invalid --allowed-origins value {origin:?}: {e}")
81 + })?;
82 + }
83 + }
84 + for method in &self.allow_methods {
85 + if method != "*" {
86 + method.parse::<Method>().map_err(|e| {
87 + anyhow::anyhow!("invalid --allowed-methods value {method:?}: {e}")
88 + })?;
89 + }
90 + }
91 + for header in &self.allow_headers {
92 + if header != "*" {
93 + header.parse::<HeaderName>().map_err(|e| {
94 + anyhow::anyhow!("invalid --allowed-headers value {header:?}: {e}")
95 + })?;
96 + }
97 + }
98 + Ok(())
99 + }
100 +}
101 +
102 /// Normalized runtime configuration for the minimal OpenAI-compatible server.
103 #[derive(Educe, Clone, PartialEq, Eq, Serialize)]
104 #[educe(Debug)]
@@ -82,6 +136,8 @@ pub struct Config {
136 pub max_logprobs: Option<i32>,
137 /// HTTP/API-server behavior switches.
138 pub api_server_options: ApiServerOptions,
139 + /// CORS settings applied to every HTTP response.
140 + pub cors: CorsConfig,
141 /// API keys accepted as bearer tokens for guarded routes.
142 #[serde(skip_serializing)]
143 #[educe(Debug(method(fmt_redacted_api_keys)))]
@@ -101,6 +157,7 @@ impl Config {
157 /// startup.
158 pub fn validate(&self) -> Result<()> {
159 vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
160 + self.cors.validate()?;
161 if let Some(max_logprobs) = self.max_logprobs
162 && max_logprobs < -1
163 {
rust/src/server/src/lib.rs
+3 -2
@@ -16,7 +16,7 @@ use std::sync::{Arc, OnceLock};
16 use anyhow::{Context as _, Result};
17 use axum::Router;
18 use axum::serve::ListenerExt as _;
19 -pub use config::{ApiServerOptions, Config, CoordinatorMode, HttpListenerMode};
19 +pub use config::{ApiServerOptions, Config, CoordinatorMode, CorsConfig, HttpListenerMode};
20 use tokio::net::TcpListener;
21 use tokio::time::{Instant, sleep_until};
22 use tokio_stream::wrappers::TcpListenerStream;
@@ -100,7 +100,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
100 AppState::new(served_model_names, chat)
101 .with_api_server_options(config.api_server_options)
102 .with_server_info(ServerInfoSnapshot::from_config(config))
103 - .with_api_keys(config.api_keys.clone()),
103 + .with_api_keys(config.api_keys.clone())
104 + .with_cors(config.cors.clone()),
105 ))
106 }
107
rust/src/server/src/middleware/cors.rs new
+141
@@ -0,0 +1,141 @@
1 +//! CORS support mirroring Python's Starlette `CORSMiddleware`.
2 +//!
3 +//! Built on `tower_http::cors::CorsLayer`, configured to reproduce Starlette's
4 +//! `CORSMiddleware` behavior for the `--allowed-origins` / `--allowed-methods` /
5 +//! `--allowed-headers` / `--allow-credentials` settings. Two intentional
6 +//! behavioral differences remain, both invisible to real clients:
7 +//!
8 +//! - A rejected preflight returns `200` (empty) rather than Starlette's
9 +//! `400 "Disallowed CORS ..."`. The browser denies the request either way
10 +//! (the disallowed `Access-Control-Allow-*` headers are simply absent), and
11 +//! tower-http makes the preflight reject decision inside its short-circuit,
12 +//! so matching the `400` would mean re-implementing the layer.
13 +//! - A bare `OPTIONS` (no `Access-Control-Request-Method`) returns `200`
14 +//! rather than `405`. No real client sends one.
15 +
16 +use std::time::Duration;
17 +
18 +use axum::extract::Request;
19 +use axum::http::{HeaderName, HeaderValue, Method, header};
20 +use axum::middleware::Next;
21 +use axum::response::Response;
22 +use tower_http::cors::{AllowHeaders, AllowMethods, AllowOrigin, CorsLayer};
23 +
24 +use crate::config::CorsConfig;
25 +
26 +/// The method set that `"*"` expands to.
27 +const ALL_METHODS: [Method; 7] = [
28 + Method::DELETE,
29 + Method::GET,
30 + Method::HEAD,
31 + Method::OPTIONS,
32 + Method::PATCH,
33 + Method::POST,
34 + Method::PUT,
35 +];
36 +
37 +/// Headers always treated as allowed (the CORS safelist).
38 +const SAFELISTED_HEADERS: [&str; 4] = [
39 + "accept",
40 + "accept-language",
41 + "content-language",
42 + "content-type",
43 +];
44 +
45 +fn is_wildcard(values: &[String]) -> bool {
46 + values.iter().any(|value| value == "*")
47 +}
48 +
49 +/// Build a `CorsLayer` from the resolved [`CorsConfig`].
50 +///
51 +/// Values are assumed valid: [`CorsConfig::validate`] runs at startup before
52 +/// the router is built.
53 +pub fn cors_layer(cfg: &CorsConfig) -> CorsLayer {
54 + let wildcard_origins = is_wildcard(&cfg.allow_origins);
55 +
56 + let allow_origin = if wildcard_origins {
57 + if cfg.allow_credentials {
58 + // `*` with credentials is illegal, so reflect the request origin
59 + // instead; this also avoids tower-http's wildcard+credentials panic.
60 + AllowOrigin::mirror_request()
61 + } else {
62 + AllowOrigin::any()
63 + }
64 + } else {
65 + AllowOrigin::list(
66 + cfg.allow_origins
67 + .iter()
68 + .map(|origin| origin.parse::<HeaderValue>().expect("validated origin"))
69 + .collect::<Vec<_>>(),
70 + )
71 + };
72 +
73 + // Expand `*` to an explicit list rather than `Any`, so we emit the method
74 + // names (not `*`) and never hit tower-http's `Any`+credentials panic.
75 + let allow_methods = if is_wildcard(&cfg.allow_methods) {
76 + AllowMethods::list(ALL_METHODS)
77 + } else {
78 + AllowMethods::list(
79 + cfg.allow_methods
80 + .iter()
81 + .map(|method| method.parse::<Method>().expect("validated method"))
82 + .collect::<Vec<_>>(),
83 + )
84 + };
85 +
86 + let allow_headers = if is_wildcard(&cfg.allow_headers) {
87 + // `*` mirrors the requested headers.
88 + AllowHeaders::mirror_request()
89 + } else {
90 + // Union the safelisted headers, lowercased and sorted.
91 + let mut names: Vec<String> = SAFELISTED_HEADERS.iter().map(|s| s.to_string()).collect();
92 + names.extend(cfg.allow_headers.iter().map(|h| h.to_ascii_lowercase()));
93 + names.sort();
94 + names.dedup();
95 + AllowHeaders::list(
96 + names
97 + .iter()
98 + .map(|header| header.parse::<HeaderName>().expect("validated header"))
99 + .collect::<Vec<_>>(),
100 + )
101 + };
102 +
103 + // Emit `Vary: Origin` only when the allow-origin is dynamic (explicit
104 + // origins, or credentials); the wildcard + no-credentials case emits no
105 + // `Vary` at all, and an empty list disables the header here.
106 + let vary: Vec<HeaderName> = if !wildcard_origins || cfg.allow_credentials {
107 + vec![header::ORIGIN]
108 + } else {
109 + vec![]
110 + };
111 +
112 + CorsLayer::new()
113 + .allow_origin(allow_origin)
114 + .allow_methods(allow_methods)
115 + .allow_headers(allow_headers)
116 + .allow_credentials(cfg.allow_credentials)
117 + .max_age(Duration::from_secs(600))
118 + .vary(vary)
119 +}
120 +
121 +/// Strip CORS response headers when the request carried no `Origin`.
122 +///
123 +/// A request without an `Origin` should carry no CORS headers, but tower-http
124 +/// emits `Vary` and `Access-Control-Allow-*` unconditionally. Removing them on
125 +/// no-`Origin` requests keeps non-CORS responses (e.g. `/health`, plain `curl`)
126 +/// clean.
127 +pub async fn strip_cors_on_no_origin(req: Request, next: Next) -> Response {
128 + let had_origin = req.headers().contains_key(header::ORIGIN);
129 + let mut response = next.run(req).await;
130 + if !had_origin {
131 + let headers = response.headers_mut();
132 + headers.remove(header::VARY);
133 + headers.remove(header::ACCESS_CONTROL_ALLOW_ORIGIN);
134 + headers.remove(header::ACCESS_CONTROL_ALLOW_CREDENTIALS);
135 + headers.remove(header::ACCESS_CONTROL_ALLOW_METHODS);
136 + headers.remove(header::ACCESS_CONTROL_ALLOW_HEADERS);
137 + headers.remove(header::ACCESS_CONTROL_MAX_AGE);
138 + headers.remove(header::ACCESS_CONTROL_EXPOSE_HEADERS);
139 + }
140 + response
141 +}
rust/src/server/src/middleware/mod.rs
+2
@@ -1,9 +1,11 @@
1 mod auth;
2 +mod cors;
3 mod load;
4 mod metrics;
5 mod request_id;
6
7 pub use auth::authenticate_api_key;
8 +pub use cors::{cors_layer, strip_cors_on_no_origin};
9 pub use load::track_server_load;
10 pub use metrics::track_http_metrics;
11 pub use request_id::set_request_id_header;
rust/src/server/src/routes.rs
+3 -1
@@ -108,7 +108,9 @@ fn build_router_with_options(
108 state.clone(),
109 middleware::track_server_load,
110 ))
111 - .layer(from_fn(middleware::track_http_metrics));
111 + .layer(from_fn(middleware::track_http_metrics))
112 + .layer(middleware::cors_layer(&state.cors))
113 + .layer(from_fn(middleware::strip_cors_on_no_origin));
114
115 if enable_api_key_auth {
116 router = router.layer(from_fn_with_state(
rust/src/server/src/routes/tests.rs
+297 -1
@@ -43,7 +43,7 @@ use zeromq::prelude::{SocketRecv, SocketSend};
43 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
44
45 use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
46 -use crate::config::ApiServerOptions;
46 +use crate::config::{ApiServerOptions, CorsConfig};
47 use crate::state::AppState;
48
49 fn request_output(
@@ -819,6 +819,35 @@ async fn test_app_with_api_keys(api_keys: Vec<String>) -> (axum::Router, MockEng
819 (app, engine_task)
820 }
821
822 +async fn test_app_with_cors_and_keys(
823 + cors: CorsConfig,
824 + api_keys: Vec<String>,
825 +) -> (axum::Router, MockEngineTask) {
826 + let (chat, engine_task) = test_models_with_engine_outputs_and_backend(
827 + b"engine-openai-cors",
828 + default_stream_output_specs(),
829 + Arc::new(FakeChatBackend::new()),
830 + )
831 + .await;
832 + let app = build_router(Arc::new(
833 + AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat)
834 + .with_cors(cors)
835 + .with_api_keys(api_keys),
836 + ));
837 + (app, engine_task)
838 +}
839 +
840 +async fn test_app_with_cors(cors: CorsConfig) -> (axum::Router, MockEngineTask) {
841 + test_app_with_cors_and_keys(cors, vec![]).await
842 +}
843 +
844 +fn header_value<'a>(response: &'a axum::response::Response, name: &str) -> Option<&'a str> {
845 + response
846 + .headers()
847 + .get(name)
848 + .map(|value| value.to_str().expect("header is valid utf-8"))
849 +}
850 +
851 async fn test_health_app_with_engine_script<F>(
852 script: F,
853 ) -> (axum::Router, Arc<AppState>, MockEngineTask)
@@ -1212,6 +1241,273 @@ async fn api_key_auth_allows_unguarded_route_without_token() {
1241 assert_eq!(response.status(), StatusCode::OK);
1242 }
1243
1244 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1245 +#[serial]
1246 +async fn cors_default_simple_request_allows_any_origin() {
1247 + let (mut app, _engine_task) = test_app_with_cors(CorsConfig::default()).await;
1248 + let response = app
1249 + .call(
1250 + Request::builder()
1251 + .method("GET")
1252 + .uri("/v1/models")
1253 + .header("origin", "http://example.com")
1254 + .body(Body::empty())
1255 + .expect("build request"),
1256 + )
1257 + .await
1258 + .expect("call app");
1259 +
1260 + assert_eq!(response.status(), StatusCode::OK);
1261 + assert_eq!(
1262 + header_value(&response, "access-control-allow-origin"),
1263 + Some("*")
1264 + );
1265 + // Wildcard origins without credentials emit no `Vary` (Starlette parity).
1266 + assert_eq!(header_value(&response, "vary"), None);
1267 +}
1268 +
1269 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1270 +#[serial]
1271 +async fn cors_default_preflight_returns_explicit_methods_and_max_age() {
1272 + let (mut app, _engine_task) = test_app_with_cors(CorsConfig::default()).await;
1273 + let response = app
1274 + .call(
1275 + Request::builder()
1276 + .method("OPTIONS")
1277 + .uri("/v1/chat/completions")
1278 + .header("origin", "http://example.com")
1279 + .header("access-control-request-method", "POST")
1280 + .header("access-control-request-headers", "content-type")
1281 + .body(Body::empty())
1282 + .expect("build request"),
1283 + )
1284 + .await
1285 + .expect("call app");
1286 +
1287 + assert_eq!(response.status(), StatusCode::OK);
1288 + // `*` methods expand to the explicit method list, matching Starlette
1289 + // (never the literal `*`).
1290 + assert_eq!(
1291 + header_value(&response, "access-control-allow-methods"),
1292 + Some("DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT")
1293 + );
1294 + assert_eq!(
1295 + header_value(&response, "access-control-max-age"),
1296 + Some("600")
1297 + );
1298 + // `*` headers mirror the requested headers.
1299 + assert_eq!(
1300 + header_value(&response, "access-control-allow-headers"),
1301 + Some("content-type")
1302 + );
1303 +}
1304 +
1305 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1306 +#[serial]
1307 +async fn cors_no_origin_request_has_no_cors_headers() {
1308 + let (mut app, _engine_task) = test_app_with_cors(CorsConfig::default()).await;
1309 + let response = app
1310 + .call(
1311 + Request::builder()
1312 + .method("GET")
1313 + .uri("/health")
1314 + .body(Body::empty())
1315 + .expect("build request"),
1316 + )
1317 + .await
1318 + .expect("call app");
1319 +
1320 + assert_eq!(response.status(), StatusCode::OK);
1321 + assert_eq!(header_value(&response, "access-control-allow-origin"), None);
1322 + assert_eq!(header_value(&response, "vary"), None);
1323 +}
1324 +
1325 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1326 +#[serial]
1327 +async fn cors_explicit_origin_allowed_reflects_origin_with_vary() {
1328 + let cors = CorsConfig {
1329 + allow_origins: vec!["http://allowed.com".to_string()],
1330 + ..CorsConfig::default()
1331 + };
1332 + let (mut app, _engine_task) = test_app_with_cors(cors).await;
1333 + let response = app
1334 + .call(
1335 + Request::builder()
1336 + .method("GET")
1337 + .uri("/v1/models")
1338 + .header("origin", "http://allowed.com")
1339 + .body(Body::empty())
1340 + .expect("build request"),
1341 + )
1342 + .await
1343 + .expect("call app");
1344 +
1345 + assert_eq!(
1346 + header_value(&response, "access-control-allow-origin"),
1347 + Some("http://allowed.com")
1348 + );
1349 + assert_eq!(header_value(&response, "vary"), Some("origin"));
1350 +}
1351 +
1352 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1353 +#[serial]
1354 +async fn cors_explicit_origin_disallowed_omits_allow_origin() {
1355 + let cors = CorsConfig {
1356 + allow_origins: vec!["http://allowed.com".to_string()],
1357 + ..CorsConfig::default()
1358 + };
1359 + let (mut app, _engine_task) = test_app_with_cors(cors).await;
1360 + let response = app
1361 + .call(
1362 + Request::builder()
1363 + .method("GET")
1364 + .uri("/v1/models")
1365 + .header("origin", "http://evil.com")
1366 + .body(Body::empty())
1367 + .expect("build request"),
1368 + )
1369 + .await
1370 + .expect("call app");
1371 +
1372 + assert_eq!(header_value(&response, "access-control-allow-origin"), None);
1373 +}
1374 +
1375 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1376 +#[serial]
1377 +async fn cors_wildcard_with_credentials_reflects_origin_without_panic() {
1378 + let cors = CorsConfig {
1379 + allow_credentials: true,
1380 + ..CorsConfig::default()
1381 + };
1382 + let (mut app, _engine_task) = test_app_with_cors(cors).await;
1383 + let response = app
1384 + .call(
1385 + Request::builder()
1386 + .method("GET")
1387 + .uri("/v1/models")
1388 + .header("origin", "http://example.com")
1389 + .body(Body::empty())
1390 + .expect("build request"),
1391 + )
1392 + .await
1393 + .expect("call app");
1394 +
1395 + // `*` + credentials reflects the request origin instead of `*` (Starlette
1396 + // parity, and avoids tower-http's wildcard+credentials panic).
1397 + assert_eq!(
1398 + header_value(&response, "access-control-allow-origin"),
1399 + Some("http://example.com")
1400 + );
1401 + assert_eq!(
1402 + header_value(&response, "access-control-allow-credentials"),
1403 + Some("true")
1404 + );
1405 + assert_eq!(header_value(&response, "vary"), Some("origin"));
1406 +}
1407 +
1408 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1409 +#[serial]
1410 +async fn cors_unauthorized_response_has_no_cors_headers() {
1411 + let (mut app, _engine_task) =
1412 + test_app_with_cors_and_keys(CorsConfig::default(), vec!["secret".to_string()]).await;
1413 + let response = app
1414 + .call(
1415 + Request::builder()
1416 + .method("GET")
1417 + .uri("/v1/models")
1418 + .header("origin", "http://example.com")
1419 + .body(Body::empty())
1420 + .expect("build request"),
1421 + )
1422 + .await
1423 + .expect("call app");
1424 +
1425 + // Auth sits outside CORS, so a 401 carries no CORS headers.
1426 + assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
1427 + assert_eq!(header_value(&response, "access-control-allow-origin"), None);
1428 +}
1429 +
1430 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1431 +#[serial]
1432 +async fn cors_preflight_bypasses_auth_and_returns_cors_headers() {
1433 + let (mut app, _engine_task) =
1434 + test_app_with_cors_and_keys(CorsConfig::default(), vec!["secret".to_string()]).await;
1435 + let response = app
1436 + .call(
1437 + Request::builder()
1438 + .method("OPTIONS")
1439 + .uri("/v1/chat/completions")
1440 + .header("origin", "http://example.com")
1441 + .header("access-control-request-method", "POST")
1442 + .body(Body::empty())
1443 + .expect("build request"),
1444 + )
1445 + .await
1446 + .expect("call app");
1447 +
1448 + assert_ne!(response.status(), StatusCode::UNAUTHORIZED);
1449 + assert_eq!(
1450 + header_value(&response, "access-control-allow-origin"),
1451 + Some("*")
1452 + );
1453 +}
1454 +
1455 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1456 +#[serial]
1457 +async fn cors_explicit_methods_preflight_returns_that_list() {
1458 + let cors = CorsConfig {
1459 + allow_methods: vec!["GET".to_string(), "POST".to_string()],
1460 + ..CorsConfig::default()
1461 + };
1462 + let (mut app, _engine_task) = test_app_with_cors(cors).await;
1463 + let response = app
1464 + .call(
1465 + Request::builder()
1466 + .method("OPTIONS")
1467 + .uri("/v1/chat/completions")
1468 + .header("origin", "http://example.com")
1469 + .header("access-control-request-method", "POST")
1470 + .body(Body::empty())
1471 + .expect("build request"),
1472 + )
1473 + .await
1474 + .expect("call app");
1475 +
1476 + // Explicit methods are emitted verbatim, not expanded and not `*`.
1477 + assert_eq!(
1478 + header_value(&response, "access-control-allow-methods"),
1479 + Some("GET,POST")
1480 + );
1481 +}
1482 +
1483 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1484 +#[serial]
1485 +async fn cors_explicit_headers_union_safelisted_headers() {
1486 + let cors = CorsConfig {
1487 + allow_headers: vec!["X-Custom".to_string()],
1488 + ..CorsConfig::default()
1489 + };
1490 + let (mut app, _engine_task) = test_app_with_cors(cors).await;
1491 + let response = app
1492 + .call(
1493 + Request::builder()
1494 + .method("OPTIONS")
1495 + .uri("/v1/chat/completions")
1496 + .header("origin", "http://example.com")
1497 + .header("access-control-request-method", "POST")
1498 + .body(Body::empty())
1499 + .expect("build request"),
1500 + )
1501 + .await
1502 + .expect("call app");
1503 +
1504 + // Explicit headers are unioned with the safelisted set, lowercased + sorted.
1505 + assert_eq!(
1506 + header_value(&response, "access-control-allow-headers"),
1507 + Some("accept,accept-language,content-language,content-type,x-custom")
1508 + );
1509 +}
1510 +
1511 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1512 #[serial]
1513 async fn version_returns_engine_vllm_version() {
rust/src/server/src/state.rs
+10 -1
@@ -9,7 +9,7 @@ use vllm_chat::ChatLlm;
9 use vllm_engine_core_client::EngineCoreClient;
10 use vllm_engine_core_client::protocol::lora::LoraRequest;
11
12 -use crate::config::ApiServerOptions;
12 +use crate::config::{ApiServerOptions, CorsConfig};
13 use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
14 use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
15
@@ -30,6 +30,8 @@ pub struct AppState {
30 pub chat: ChatLlm,
31 /// HTTP/API-server behavior switches.
32 pub api_server_options: ApiServerOptions,
33 + /// CORS settings applied to every HTTP response.
34 + pub cors: CorsConfig,
35 /// Runtime server information returned by `/server_info`, when available.
36 server_info: Option<ServerInfoSnapshot>,
37 /// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes.
@@ -58,6 +60,7 @@ impl AppState {
60 served_model_names,
61 chat,
62 api_server_options: ApiServerOptions::default(),
63 + cors: CorsConfig::default(),
64 server_info: None,
65 api_key_hashes: Vec::new(),
66 server_load: AtomicU64::new(0),
@@ -71,6 +74,12 @@ impl AppState {
74 self
75 }
76
77 + /// Set the CORS settings applied to every HTTP response.
78 + pub fn with_cors(mut self, cors: CorsConfig) -> Self {
79 + self.cors = cors;
80 + self
81 + }
82 +
83 /// Attach the runtime server information snapshot used by `/server_info`.
84 pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self {
85 self.server_info = Some(server_info);