A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][Rust Frontend] Make metrics respect --served-model-name (#45465)
Signed-off-by: reidliu41 <reid201711@gmail.com>
Reid committed
Jun 15, 2026 at 16:05 UTC
c17e2f7c84d28dfcf5e8cfcc3c5c10bd3caad8b5
2 files changed
+118
-9
rust/src/server/src/lib.rs
+39
-9
@@ -35,9 +35,24 @@ use crate::routes::build_router;
35
use crate::server_info::ServerInfoSnapshot;
36
use crate::state::AppState;
37
38
+/// Resolve the public model names accepted by the frontend.
39
+fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Vec<String> {
40
+ if served_model_name.is_empty() {
41
+ vec![model.to_string()]
42
+ } else {
43
+ served_model_name.to_vec()
44
+ }
45
+}
46
+
47
/// Build the shared application state for one configured model and one engine
48
/// client.
49
async fn build_state(config: &Config) -> Result<Arc<AppState>> {
50
+ // If no served names are specified, fall back to the backend model path so
51
+ // that the API always has at least one valid model ID. Use the same primary
52
+ // public name for frontend-side metrics labels.
53
+ let served_model_names = effective_served_model_names(&config.model, &config.served_model_name);
54
+ let metrics_model_name = served_model_names[0].clone();
55
+
56
// Load both backends from the same model metadata so they stay in sync.
57
let loaded = load_model_backends(
58
&config.model,
@@ -68,7 +83,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
83
let client = EngineCoreClient::connect(EngineCoreClientConfig {
84
transport_mode: config.transport_mode.clone(),
85
coordinator_mode,
71
- model_name: config.model.clone(),
86
+ model_name: metrics_model_name,
87
client_index: 0,
88
})
89
.await
@@ -81,14 +96,6 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
96
.with_tool_call_parser(config.tool_call_parser.clone())
97
.with_reasoning_parser(config.reasoning_parser.clone());
98
84
- // If no served names are specified, fall back to the backend model path so
85
- // that the API always has at least one valid model ID.
86
- let served_model_names = if config.served_model_name.is_empty() {
87
- vec![config.model.clone()]
88
- } else {
89
- config.served_model_name.clone()
90
- };
91
-
99
Ok(Arc::new(
100
AppState::new(served_model_names, chat)
101
.with_api_server_options(config.api_server_options)
@@ -258,3 +265,26 @@ where
265
.unwrap_or_else(|| Instant::now() + config.shutdown_timeout);
266
state.shutdown(shutdown_deadline).await
267
}
268
+
269
+#[cfg(test)]
270
+mod tests {
271
+ use super::*;
272
+
273
+ #[test]
274
+ fn effective_served_model_names_falls_back_to_backend_model() {
275
+ assert_eq!(
276
+ effective_served_model_names("backend-model", &[]),
277
+ vec!["backend-model"]
278
+ );
279
+ }
280
+
281
+ #[test]
282
+ fn effective_served_model_names_preserves_public_names() {
283
+ let served_names = vec!["public-model".to_string(), "public-alias".to_string()];
284
+
285
+ assert_eq!(
286
+ effective_served_model_names("backend-model", &served_names),
287
+ served_names
288
+ );
289
+ }
290
+}
rust/src/server/src/routes/tests.rs
+79
@@ -1677,6 +1677,85 @@ async fn http_metrics_record_list_models_requests() {
1677
);
1678
}
1679
1680
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1681
+#[serial]
1682
+async fn request_metrics_use_served_model_name_label() {
1683
+ let ipc = IpcNamespace::new().expect("create ipc namespace");
1684
+ let handshake_address = ipc.handshake_endpoint();
1685
+ let engine_id = b"engine-openai-served-model-metrics".to_vec();
1686
+
1687
+ let engine_task = MockEngineTask::new(spawn_mock_engine_task(
1688
+ handshake_address.clone(),
1689
+ engine_id.clone(),
1690
+ |dealer, push| {
1691
+ boxed_test_future(async move {
1692
+ let add = recv_engine_message(dealer).await;
1693
+ let request: EngineCoreRequest =
1694
+ rmp_serde::from_slice(&add[1]).expect("decode request");
1695
+ send_outputs(
1696
+ push,
1697
+ engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
1698
+ )
1699
+ .await;
1700
+ })
1701
+ },
1702
+ ));
1703
+
1704
+ let client = EngineCoreClient::connect(
1705
+ EngineCoreClientConfig::new_single(handshake_address)
1706
+ .with_model_name("served-model-metrics")
1707
+ .with_local_input_output_addresses(
1708
+ Some(ipc.input_endpoint()),
1709
+ Some(ipc.output_endpoint()),
1710
+ ),
1711
+ )
1712
+ .await
1713
+ .expect("connect client");
1714
+ let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
1715
+ let mut app = build_router(Arc::new(AppState::new(
1716
+ vec![
1717
+ "served-model-metrics".to_string(),
1718
+ "served-model-alias".to_string(),
1719
+ ],
1720
+ chat,
1721
+ )));
1722
+ let before = METRICS.render().unwrap();
1723
+
1724
+ let response = app
1725
+ .call(
1726
+ Request::builder()
1727
+ .method("POST")
1728
+ .uri("/v1/chat/completions")
1729
+ .header("content-type", "application/json")
1730
+ .body(Body::from(
1731
+ json!({
1732
+ "model": "served-model-alias",
1733
+ "stream": false,
1734
+ "messages": [{"role": "user", "content": "hello"}]
1735
+ })
1736
+ .to_string(),
1737
+ ))
1738
+ .expect("build request"),
1739
+ )
1740
+ .await
1741
+ .expect("call app");
1742
+
1743
+ assert_eq!(response.status(), StatusCode::OK);
1744
+ let _ = to_bytes(response.into_body(), usize::MAX).await.unwrap();
1745
+
1746
+ let after = METRICS.render().unwrap();
1747
+ assert_eq!(
1748
+ metric_delta(
1749
+ &before,
1750
+ &after,
1751
+ "vllm:request_success_total",
1752
+ Some("model_name=\"served-model-metrics\",engine=\"0\",finished_reason=\"stop\""),
1753
+ ),
1754
+ 1.0
1755
+ );
1756
+ engine_task.await.expect("mock engine task");
1757
+}
1758
+
1759
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1760
#[serial]
1761
async fn wrong_model_returns_not_found() {