[Rust Frontend] Return model metadata fields in /v1/models (#45950)
Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>
Tahsin Tunan committed
Jun 18, 2026 at 16:29 UTC
7299e6509ef8b9d27e86c4f2315e1ec5628ca426
8 files changed
+153
-26
rust/Cargo.lock
+1
@@ -5880,6 +5880,7 @@ dependencies = [
5880
"expect-test",
5881
"futures",
5882
"http-body",
5883
+ "indexmap 2.13.0",
5884
"itertools 0.14.0",
5885
"libc",
5886
"llm-multimodal",
rust/src/server/Cargo.toml
+1
@@ -11,6 +11,7 @@ axum.workspace = true
11
educe.workspace = true
12
futures.workspace = true
13
http-body.workspace = true
14
+indexmap.workspace = true
15
itertools.workspace = true
16
libc.workspace = true
17
llm-multimodal.workspace = true
rust/src/server/src/lib.rs
+1
@@ -98,6 +98,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
98
99
Ok(Arc::new(
100
AppState::new(served_model_names, chat)
101
+ .with_model_path(config.model.clone())
102
.with_api_server_options(config.api_server_options)
103
.with_server_info(ServerInfoSnapshot::from_config(config))
104
.with_api_keys(config.api_keys.clone())
rust/src/server/src/lora.rs
+8
-11
@@ -1,6 +1,6 @@
1
-use std::collections::BTreeMap;
1
use std::sync::atomic::{AtomicU64, Ordering};
2
3
+use indexmap::IndexMap;
4
use tokio::sync::{Mutex, RwLock};
5
use vllm_engine_core_client::EngineCoreClient;
6
use vllm_engine_core_client::protocol::lora::LoraRequest;
@@ -15,8 +15,8 @@ pub(crate) struct LoraModelResolution {
15
16
/// Runtime registry for dynamically loaded LoRA adapters.
17
pub(crate) struct LoraManager {
18
- /// Dynamically loaded LoRA adapters keyed by public model name.
19
- requests: RwLock<BTreeMap<String, LoraRequest>>,
18
+ /// Dynamically loaded LoRA adapters keyed by public model name, in load order.
19
+ requests: RwLock<IndexMap<String, LoraRequest>>,
20
/// Monotonic adapter id allocator. LoRA ids are one-indexed.
21
id_counter: AtomicU64,
22
/// Serialize dynamic LoRA registry updates around engine utility calls.
@@ -51,18 +51,15 @@ pub(crate) enum UnloadLoraError {
51
impl LoraManager {
52
pub fn new() -> Self {
53
Self {
54
- requests: RwLock::new(BTreeMap::new()),
54
+ requests: RwLock::new(IndexMap::new()),
55
id_counter: AtomicU64::new(0),
56
update_lock: Mutex::new(()),
57
}
58
}
59
60
- /// Return base served model names plus dynamically loaded LoRA adapter
61
- /// names.
62
- pub async fn served_model_names(&self, base_model_names: &[String]) -> Vec<String> {
63
- let mut names = base_model_names.to_vec();
64
- names.extend(self.requests.read().await.keys().cloned());
65
- names
60
+ /// Snapshot loaded LoRA adapters in load order.
61
+ pub async fn served_lora_requests(&self) -> Vec<LoraRequest> {
62
+ self.requests.read().await.values().cloned().collect()
63
}
64
65
/// Resolve the requested model against one consistent LoRA registry
@@ -163,6 +160,6 @@ impl LoraManager {
160
});
161
}
162
166
- Ok(self.requests.write().await.remove(lora_name).unwrap_or(lora_request))
163
+ Ok(self.requests.write().await.shift_remove(lora_name).unwrap_or(lora_request))
164
}
165
}
rust/src/server/src/routes/openai/models.rs
+32
-11
@@ -1,4 +1,5 @@
1
use std::sync::Arc;
2
+use std::time::{SystemTime, UNIX_EPOCH};
3
4
use axum::Json;
5
use axum::extract::State;
@@ -6,19 +7,39 @@ use axum::extract::State;
7
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject};
8
use crate::state::AppState;
9
9
-/// Return all configured served model names in OpenAI `list models` format.
10
+// Frontend marker; Python uses "vllm".
11
+const OWNED_BY: &str = "vllm-frontend-rs";
12
+
13
+/// Base cards carry `max_model_len` and `root` = model path; LoRA cards carry
14
+/// `root` = adapter path and `parent` = base model. LoRA cards follow load order.
15
pub async fn list_models(State(state): State<Arc<AppState>>) -> Json<ListModelsResponse> {
11
- let model_names = state.served_model_names_with_loras().await;
16
+ let created = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() as i64;
17
+ let max_model_len = state.chat.engine_core_client().max_model_len();
18
+ let model_path = state.model_path().map(str::to_string);
19
+
20
+ let base_cards = state.served_model_names().iter().map(|name| ModelObject {
21
+ id: name.clone(),
22
+ object: "model".to_string(),
23
+ created,
24
+ owned_by: OWNED_BY.to_string(),
25
+ root: Some(model_path.clone().unwrap_or_else(|| name.clone())),
26
+ parent: None,
27
+ max_model_len: Some(max_model_len),
28
+ });
29
+
30
+ let primary = state.primary_model_name().to_string();
31
+ let lora_cards = state.served_lora_requests().await.into_iter().map(|lora| ModelObject {
32
+ id: lora.lora_name,
33
+ object: "model".to_string(),
34
+ created,
35
+ owned_by: OWNED_BY.to_string(),
36
+ root: Some(lora.lora_path),
37
+ parent: Some(lora.base_model_name.unwrap_or_else(|| primary.clone())),
38
+ max_model_len: None,
39
+ });
40
+
41
Json(ListModelsResponse {
42
object: "list".to_string(),
14
- data: model_names
15
- .into_iter()
16
- .map(|name| ModelObject {
17
- id: name,
18
- object: "model".to_string(),
19
- created: 0,
20
- owned_by: "vllm-frontend-rs".to_string(),
21
- })
22
- .collect(),
43
+ data: base_cards.chain(lora_cards).collect(),
44
})
45
}
rust/src/server/src/routes/openai/utils/types.rs
+6
@@ -457,6 +457,12 @@ pub struct ModelObject {
457
pub object: String,
458
pub created: i64,
459
pub owned_by: String,
460
+ /// Backend model path (base cards) or adapter path (LoRA cards).
461
+ pub root: Option<String>,
462
+ /// Base model a LoRA adapter derives from; `null` for base models.
463
+ pub parent: Option<String>,
464
+ /// Maximum context length; `null` for LoRA adapter cards.
465
+ pub max_model_len: Option<u32>,
466
}
467
468
/// Response body for `GET /v1/models`.
rust/src/server/src/routes/tests.rs
+87
@@ -1107,6 +1107,93 @@ async fn list_models_returns_configured_model() {
1107
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
1108
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1109
assert_eq!(json["data"][0]["id"], "Qwen/Qwen1.5-0.5B-Chat");
1110
+ // No model path configured: `root` falls back to the served name.
1111
+ assert_eq!(json["data"][0]["root"], "Qwen/Qwen1.5-0.5B-Chat");
1112
+}
1113
+
1114
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1115
+#[serial]
1116
+async fn list_models_base_card_includes_metadata() {
1117
+ let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
1118
+ b"engine-openai-models-meta",
1119
+ default_stream_output_specs(),
1120
+ Arc::new(FakeChatBackend::new()),
1121
+ )
1122
+ .await;
1123
+ // `id` is the served alias; `root` is the underlying model path.
1124
+ let mut app = build_router(Arc::new(
1125
+ AppState::new(vec!["public-alias".to_string()], chat)
1126
+ .with_model_path("org/backend-model".to_string()),
1127
+ ));
1128
+
1129
+ let response = app
1130
+ .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1131
+ .await
1132
+ .expect("call app");
1133
+ assert_eq!(response.status(), StatusCode::OK);
1134
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
1135
+ let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1136
+
1137
+ let card = json["data"][0].as_object().expect("card object");
1138
+ assert_eq!(card["id"], "public-alias");
1139
+ assert_eq!(card["owned_by"], "vllm-frontend-rs");
1140
+ assert_eq!(card["root"], "org/backend-model");
1141
+ assert!(card["max_model_len"].as_u64().expect("max_model_len") > 0);
1142
+ assert!(card["created"].as_i64().expect("created") > 0);
1143
+ // `parent` must be emitted as null, not omitted.
1144
+ assert!(card.contains_key("parent") && card["parent"].is_null());
1145
+}
1146
+
1147
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1148
+#[serial]
1149
+async fn list_models_lists_loras_in_load_order() {
1150
+ // Load out of lexicographic order; the list must preserve load order, not sort.
1151
+ let (mut app, _engine_task) = test_admin_app_with_engine_script(|dealer, push| {
1152
+ boxed_test_future(async move {
1153
+ for _ in 0..2 {
1154
+ let utility = recv_engine_message(dealer).await;
1155
+ let payload = decode_value(&utility[1]).expect("decode utility payload");
1156
+ let call_id =
1157
+ payload.as_array().expect("utility array")[1].as_u64().expect("call id");
1158
+ send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await;
1159
+ }
1160
+ })
1161
+ })
1162
+ .await;
1163
+
1164
+ for name in ["zebra", "alpha"] {
1165
+ let path = format!("org/{name}");
1166
+ let response = app
1167
+ .call(
1168
+ Request::builder()
1169
+ .method("POST")
1170
+ .uri("/v1/load_lora_adapter")
1171
+ .header("content-type", "application/json")
1172
+ .body(Body::from(
1173
+ json!({ "lora_name": name, "lora_path": path }).to_string(),
1174
+ ))
1175
+ .expect("build request"),
1176
+ )
1177
+ .await
1178
+ .expect("call app");
1179
+ assert_eq!(response.status(), StatusCode::OK);
1180
+ }
1181
+
1182
+ let models = app
1183
+ .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1184
+ .await
1185
+ .expect("call app");
1186
+ let body = to_bytes(models.into_body(), usize::MAX).await.expect("read body");
1187
+ let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1188
+
1189
+ assert_eq!(json["data"][0]["id"], "Qwen/Qwen1.5-0.5B-Chat");
1190
+ assert_eq!(json["data"][1]["id"], "zebra");
1191
+ assert_eq!(json["data"][2]["id"], "alpha");
1192
+ // `max_model_len` must be emitted as null on LoRA cards, not omitted.
1193
+ let lora_card = json["data"][1].as_object().expect("lora card object");
1194
+ assert_eq!(lora_card["root"], "org/zebra");
1195
+ assert_eq!(lora_card["parent"], "Qwen/Qwen1.5-0.5B-Chat");
1196
+ assert!(lora_card.contains_key("max_model_len") && lora_card["max_model_len"].is_null());
1197
}
1198
1199
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
rust/src/server/src/state.rs
+17
-4
@@ -40,6 +40,8 @@ pub struct AppState {
40
server_load: AtomicU64,
41
/// Dynamic LoRA adapter registry.
42
lora_manager: LoraManager,
43
+ /// Backend model path reported as `root` for base-model cards.
44
+ model_path: Option<String>,
45
}
46
47
impl AppState {
@@ -65,6 +67,7 @@ impl AppState {
67
api_key_hashes: Vec::new(),
68
server_load: AtomicU64::new(0),
69
lora_manager: LoraManager::new(),
70
+ model_path: None,
71
}
72
}
73
@@ -80,6 +83,12 @@ impl AppState {
83
self
84
}
85
86
+ /// Set the backend model path reported as `root` for base-model cards.
87
+ pub fn with_model_path(mut self, model_path: String) -> Self {
88
+ self.model_path = Some(model_path);
89
+ self
90
+ }
91
+
92
/// Attach the runtime server information snapshot used by `/server_info`.
93
pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self {
94
self.server_info = Some(server_info);
@@ -123,10 +132,14 @@ impl AppState {
132
&self.served_model_names
133
}
134
126
- /// Return base served model names plus dynamically loaded LoRA adapter
127
- /// names.
128
- pub async fn served_model_names_with_loras(&self) -> Vec<String> {
129
- self.lora_manager.served_model_names(&self.served_model_names).await
135
+ /// Backend model path reported as `root` for base-model cards, if known.
136
+ pub fn model_path(&self) -> Option<&str> {
137
+ self.model_path.as_deref()
138
+ }
139
+
140
+ /// Snapshot the loaded LoRA adapters in load order, for `/v1/models` cards.
141
+ pub async fn served_lora_requests(&self) -> Vec<LoraRequest> {
142
+ self.lora_manager.served_lora_requests().await
143
}
144
145
/// Resolve the requested model against one dynamic LoRA registry snapshot.