A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 02:15 UTC
[Rust Frontend] Add /server_info to Rust frontend (#43942)
Signed-off-by: xunzhuo <xunzhuo@vllm-semantic-router.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Xunzhuo committed
Jun 3, 2026 at 19:30 UTC
309385a359b1722ed0f2720540421a3fba08a9ac
11 files changed
+274
-15
rust/src/chat/src/parser/mod.rs
+2
-2
@@ -6,10 +6,10 @@ use std::convert::Infallible;
6
use std::fmt;
7
use std::str::FromStr;
8
9
-use serde_with::DeserializeFromStr;
9
+use serde_with::{DeserializeFromStr, SerializeDisplay};
10
11
/// Specify which reasoning or tool-call parser implementation to use.
12
-#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr)]
12
+#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
13
pub enum ParserSelection {
14
/// Use model-based auto-detection.
15
#[default]
rust/src/chat/src/renderer/hf/format.rs
+2
-2
@@ -5,7 +5,7 @@ use std::str::FromStr;
5
use minijinja::machinery::ast::{Expr, ForLoop, Set, Stmt};
6
use minijinja::machinery::{WhitespaceConfig, parse};
7
use minijinja::syntax::SyntaxConfig;
8
-use serde_with::DeserializeFromStr;
8
+use serde_with::{DeserializeFromStr, SerializeDisplay};
9
10
/// Chat template content format.
11
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -18,7 +18,7 @@ pub enum ChatTemplateContentFormat {
18
}
19
20
/// Configurable chat-template content format selection.
21
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
21
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
22
pub enum ChatTemplateContentFormatOption {
23
/// Detect the format from the template source.
24
#[default]
rust/src/chat/src/renderer/selection.rs
+2
-2
@@ -1,10 +1,10 @@
1
use std::fmt;
2
use std::str::FromStr;
3
4
-use serde_with::DeserializeFromStr;
4
+use serde_with::{DeserializeFromStr, SerializeDisplay};
5
6
/// Specify which chat renderer implementation to use.
7
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)]
7
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
8
pub enum RendererSelection {
9
/// Use model-based auto-detection.
10
#[default]
rust/src/engine-core-client/src/client.rs
+2
-1
@@ -2,6 +2,7 @@ use std::sync::Arc;
2
use std::time::Duration;
3
4
use futures::future::{join_all, try_join_all};
5
+use serde::Serialize;
6
use tokio::sync::mpsc;
7
use tokio_util::task::AbortOnDropHandle;
8
use tracing::{debug, info, trace};
@@ -23,7 +24,7 @@ pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
24
25
/// How the frontend acquires its request/response transport with Python
26
/// `EngineCoreProc`s.
26
-#[derive(Debug, Clone, PartialEq, Eq)]
27
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
28
pub enum TransportMode {
29
/// The Rust process owns the startup handshake and allocates or binds the
30
/// frontend transport addresses itself before replying to engine
rust/src/server/src/config.rs
+4
-3
@@ -2,12 +2,13 @@ use std::collections::HashMap;
2
use std::time::Duration;
3
4
use anyhow::Result;
5
+use serde::Serialize;
6
use serde_json::Value;
7
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
8
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
9
10
/// How the HTTP server obtains its listening socket.
10
-#[derive(Debug, Clone, PartialEq, Eq)]
11
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
12
pub enum HttpListenerMode {
13
/// Bind a fresh TCP listener on the given host/port.
14
BindTcp { host: String, port: u16 },
@@ -20,7 +21,7 @@ pub enum HttpListenerMode {
21
22
/// Which coordinator implementation should be active when one is present for a
23
/// frontend client.
23
-#[derive(Debug, Clone, PartialEq, Eq)]
24
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
25
pub enum CoordinatorMode {
26
/// Do not run a coordinator at all.
27
None,
@@ -32,7 +33,7 @@ pub enum CoordinatorMode {
33
}
34
35
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
35
-#[derive(Debug, Clone, PartialEq, Eq)]
36
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
37
pub struct Config {
38
/// Frontend-to-engine transport setup.
39
pub transport_mode: TransportMode,
rust/src/server/src/lib.rs
+4
-1
@@ -7,6 +7,7 @@ mod listener;
7
mod lora;
8
mod middleware;
9
mod routes;
10
+mod server_info;
11
mod state;
12
mod utils;
13
@@ -30,6 +31,7 @@ use vllm_text::TextLlm;
31
32
use crate::listener::Listener;
33
use crate::routes::build_router;
34
+use crate::server_info::ServerInfoSnapshot;
35
use crate::state::AppState;
36
37
/// Build the shared application state for one configured model and one engine
@@ -88,7 +90,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
90
Ok(Arc::new(
91
AppState::new(served_model_names, chat)
92
.with_log_requests(config.enable_log_requests)
91
- .with_request_id_headers(config.enable_request_id_headers),
93
+ .with_request_id_headers(config.enable_request_id_headers)
94
+ .with_server_info(ServerInfoSnapshot::from_config(config)),
95
))
96
}
97
rust/src/server/src/routes.rs
+2
@@ -6,6 +6,7 @@ mod load;
6
mod lora;
7
mod metrics;
8
pub(crate) mod openai;
9
+mod server_info;
10
mod sleep;
11
mod version;
12
@@ -89,6 +90,7 @@ fn build_router_with_options(
90
.route("/sleep", post(sleep::sleep))
91
.route("/wake_up", post(sleep::wake_up))
92
.route("/is_sleeping", get(sleep::is_sleeping))
93
+ .route("/server_info", get(server_info::server_info))
94
}
95
96
let enable_request_id_headers = state.enable_request_id_headers;
rust/src/server/src/routes/server_info.rs
new
+47
@@ -0,0 +1,47 @@
1
+use std::sync::Arc;
2
+
3
+use axum::Json;
4
+use axum::extract::{Query, State};
5
+use axum::http::StatusCode;
6
+use axum::response::{IntoResponse, Response};
7
+use serde::Deserialize;
8
+
9
+use crate::server_info::ServerInfoConfigFormat;
10
+use crate::state::AppState;
11
+
12
+#[derive(Debug, Clone, Copy, Deserialize)]
13
+#[serde(rename_all = "lowercase")]
14
+enum ConfigFormat {
15
+ Text,
16
+ Json,
17
+}
18
+
19
+impl From<ConfigFormat> for ServerInfoConfigFormat {
20
+ fn from(value: ConfigFormat) -> Self {
21
+ match value {
22
+ ConfigFormat::Text => Self::Text,
23
+ ConfigFormat::Json => Self::Json,
24
+ }
25
+ }
26
+}
27
+
28
+fn default_config_format() -> ConfigFormat {
29
+ ConfigFormat::Text
30
+}
31
+
32
+#[derive(Debug, Deserialize)]
33
+pub(crate) struct ServerInfoParams {
34
+ #[serde(default = "default_config_format")]
35
+ config_format: ConfigFormat,
36
+}
37
+
38
+/// Get server configuration and environment metadata.
39
+pub async fn server_info(
40
+ State(state): State<Arc<AppState>>,
41
+ Query(params): Query<ServerInfoParams>,
42
+) -> Response {
43
+ match state.server_info_response(params.config_format.into()) {
44
+ Some(response) => Json(response).into_response(),
45
+ None => StatusCode::NOT_FOUND.into_response(),
46
+ }
47
+}
rust/src/server/src/routes/tests.rs
+45
-4
@@ -742,16 +742,23 @@ async fn test_chat_with_engine_outputs(
742
}
743
744
async fn test_app() -> axum::Router {
745
+ test_app_with_dev_mode(false).await
746
+}
747
+
748
+async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
749
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
750
b"engine-openai",
751
default_stream_output_specs(),
752
Arc::new(FakeChatBackend::new()),
753
)
754
.await;
751
- build_router(Arc::new(AppState::new(
752
- vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
753
- chat,
754
- )))
755
+ build_router_with_dev_mode(
756
+ Arc::new(AppState::new(
757
+ vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
758
+ chat,
759
+ )),
760
+ dev_mode_enabled,
761
+ )
762
}
763
764
async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
@@ -1087,6 +1094,23 @@ async fn version_returns_engine_vllm_version() {
1094
);
1095
}
1096
1097
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1098
+#[serial]
1099
+async fn server_info_endpoint_is_dev_mode_only() {
1100
+ let mut app = test_app().await;
1101
+ let response = app
1102
+ .call(
1103
+ Request::builder()
1104
+ .uri("/server_info")
1105
+ .body(Body::empty())
1106
+ .expect("build request"),
1107
+ )
1108
+ .await
1109
+ .expect("call app");
1110
+
1111
+ assert_eq!(response.status(), StatusCode::NOT_FOUND);
1112
+}
1113
+
1114
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1115
#[serial]
1116
async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
@@ -1302,6 +1326,23 @@ async fn load_lora_adapter_registers_model_and_forwards_lora_request() {
1326
engine_task.finish().await;
1327
}
1328
1329
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1330
+#[serial]
1331
+async fn server_info_endpoint_returns_not_found_without_snapshot() {
1332
+ let mut app = test_app_with_dev_mode(true).await;
1333
+ let response = app
1334
+ .call(
1335
+ Request::builder()
1336
+ .uri("/server_info")
1337
+ .body(Body::empty())
1338
+ .expect("build request"),
1339
+ )
1340
+ .await
1341
+ .expect("call app");
1342
+
1343
+ assert_eq!(response.status(), StatusCode::NOT_FOUND);
1344
+}
1345
+
1346
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1347
#[serial]
1348
async fn unload_lora_adapter_rejects_mismatched_lora_int_id() {
rust/src/server/src/server_info.rs
new
+144
@@ -0,0 +1,144 @@
1
+use std::collections::BTreeMap;
2
+
3
+use serde_json::{Value, json};
4
+
5
+use crate::config::Config;
6
+
7
+const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] =
8
+ &["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"];
9
+
10
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11
+pub(crate) enum ServerInfoConfigFormat {
12
+ Text,
13
+ Json,
14
+}
15
+
16
+/// Snapshot returned by `/server_info`.
17
+#[derive(Debug, Clone)]
18
+pub(crate) struct ServerInfoSnapshot {
19
+ vllm_config_text: String,
20
+ vllm_config_json: Value,
21
+ vllm_env: BTreeMap<String, String>,
22
+ system_env: BTreeMap<String, String>,
23
+}
24
+
25
+impl ServerInfoSnapshot {
26
+ /// Capture the runtime configuration fields available to the Rust frontend.
27
+ pub(crate) fn from_config(config: &Config) -> Self {
28
+ let vllm_config_json =
29
+ serde_json::to_value(config).expect("server info value must serialize");
30
+
31
+ Self {
32
+ vllm_config_text: render_config_text(&vllm_config_json),
33
+ vllm_config_json,
34
+ vllm_env: collect_vllm_env(),
35
+ system_env: collect_system_env(),
36
+ }
37
+ }
38
+
39
+ pub(crate) fn response(&self, config_format: ServerInfoConfigFormat) -> Value {
40
+ let vllm_config = match config_format {
41
+ ServerInfoConfigFormat::Text => Value::String(self.vllm_config_text.clone()),
42
+ ServerInfoConfigFormat::Json => self.vllm_config_json.clone(),
43
+ };
44
+
45
+ json!({
46
+ "vllm_config": vllm_config,
47
+ "vllm_env": self.vllm_env.clone(),
48
+ "system_env": self.system_env.clone(),
49
+ })
50
+ }
51
+}
52
+
53
+fn render_config_text(config: &Value) -> String {
54
+ match config {
55
+ Value::Object(fields) => fields
56
+ .iter()
57
+ .map(|(key, value)| format!("{key}={}", render_config_text_value(value)))
58
+ .collect::<Vec<_>>()
59
+ .join("\n"),
60
+ _ => render_config_text_value(config),
61
+ }
62
+}
63
+
64
+fn render_config_text_value(value: &Value) -> String {
65
+ match value {
66
+ Value::Null => "None".to_string(),
67
+ Value::String(value) => value.clone(),
68
+ _ => value.to_string(),
69
+ }
70
+}
71
+
72
+fn collect_vllm_env() -> BTreeMap<String, String> {
73
+ std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect()
74
+}
75
+
76
+fn is_public_vllm_env_key(key: &str) -> bool {
77
+ let key = key.to_ascii_uppercase();
78
+ key.starts_with("VLLM_")
79
+ && !SENSITIVE_VLLM_ENV_PATTERNS.iter().any(|pattern| key.contains(pattern))
80
+}
81
+
82
+fn collect_system_env() -> BTreeMap<String, String> {
83
+ BTreeMap::from([
84
+ ("arch".to_string(), std::env::consts::ARCH.to_string()),
85
+ ("family".to_string(), std::env::consts::FAMILY.to_string()),
86
+ ("os".to_string(), std::env::consts::OS.to_string()),
87
+ ])
88
+}
89
+
90
+#[cfg(test)]
91
+mod tests {
92
+ use std::collections::BTreeSet;
93
+
94
+ use serde_json::{Value, json};
95
+
96
+ use super::{is_public_vllm_env_key, render_config_text};
97
+
98
+ #[test]
99
+ fn render_config_text_formats_config_snapshot() {
100
+ let rendered = render_config_text(&json!({
101
+ "model": "test-model",
102
+ "served_model_name": ["served-model"],
103
+ "chat_template": null,
104
+ "enable_log_requests": true,
105
+ }));
106
+ let lines = rendered.lines().collect::<BTreeSet<_>>();
107
+
108
+ assert_eq!(
109
+ lines,
110
+ BTreeSet::from([
111
+ "chat_template=None",
112
+ "enable_log_requests=true",
113
+ "model=test-model",
114
+ "served_model_name=[\"served-model\"]",
115
+ ])
116
+ );
117
+ assert_eq!(
118
+ render_config_text(&Value::String("inline".to_string())),
119
+ "inline"
120
+ );
121
+ assert_eq!(render_config_text(&Value::Null), "None");
122
+ }
123
+
124
+ #[test]
125
+ fn server_info_env_filter_excludes_sensitive_vllm_keys() {
126
+ for key in [
127
+ "VLLM_API_KEY",
128
+ "VLLM_AUTH_TOKEN",
129
+ "VLLM_SECRET",
130
+ "VLLM_PASSWORD",
131
+ "VLLM_CREDENTIAL_FILE",
132
+ "vllm_token",
133
+ ] {
134
+ assert!(!is_public_vllm_env_key(key), "{key}");
135
+ }
136
+ }
137
+
138
+ #[test]
139
+ fn server_info_env_filter_includes_public_vllm_keys() {
140
+ assert!(is_public_vllm_env_key("VLLM_LOGGING_LEVEL"));
141
+ assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE"));
142
+ assert!(!is_public_vllm_env_key("OTHER_ENV"));
143
+ }
144
+}
rust/src/server/src/state.rs
+20
@@ -1,6 +1,7 @@
1
use std::sync::Arc;
2
use std::sync::atomic::{AtomicU64, Ordering};
3
4
+use serde_json::Value;
5
use tokio::time::{Duration, Instant, sleep_until};
6
use tracing::warn;
7
use vllm_chat::ChatLlm;
@@ -9,6 +10,8 @@ use vllm_engine_core_client::protocol::lora::LoraRequest;
10
11
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
12
13
+use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
14
+
15
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
16
17
/// Shared router state for the minimal single-model OpenAI server.
@@ -22,6 +25,8 @@ pub struct AppState {
25
pub enable_log_requests: bool,
26
/// Whether to set X-Request-Id on every HTTP response.
27
pub enable_request_id_headers: bool,
28
+ /// Runtime server information returned by `/server_info`, when available.
29
+ server_info: Option<ServerInfoSnapshot>,
30
/// Number of in-flight inference requests currently owned by this frontend.
31
server_load: AtomicU64,
32
/// Dynamic LoRA adapter registry.
@@ -47,6 +52,7 @@ impl AppState {
52
chat,
53
enable_log_requests: false,
54
enable_request_id_headers: false,
55
+ server_info: None,
56
server_load: AtomicU64::new(0),
57
lora_manager: LoraManager::new(),
58
}
@@ -64,6 +70,20 @@ impl AppState {
70
self
71
}
72
73
+ /// Attach the runtime server information snapshot used by `/server_info`.
74
+ pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self {
75
+ self.server_info = Some(server_info);
76
+ self
77
+ }
78
+
79
+ /// Build a `/server_info` response payload.
80
+ pub(crate) fn server_info_response(
81
+ &self,
82
+ config_format: ServerInfoConfigFormat,
83
+ ) -> Option<Value> {
84
+ self.server_info.as_ref().map(|server_info| server_info.response(config_format))
85
+ }
86
+
87
/// The primary model name echoed back in API responses (the first served
88
/// name).
89
pub fn primary_model_name(&self) -> &str {