@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1521173c17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 12:15 UTC

[Rust Frontend] Add `/version` endpoint using engine-reported value (#43854)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed May 29, 2026 at 08:32 UTC 1521173c170f395b697bade2339c7946a68aa654
17 files changed +158 -194
rust/src/chat/src/lib.rs
+2 -2
@@ -93,7 +93,7 @@ pub struct ChatLlm {
93 text: TextLlm,
94 backend: DynChatBackend,
95 /// Effective model dtype reported by the engine.
96 - model_dtype: Option<ModelDtype>,
96 + model_dtype: ModelDtype,
97 /// Tool-call parser selection.
98 tool_call_parser: ParserSelection,
99 /// Reasoning parser selection.
@@ -135,7 +135,7 @@ impl ChatLlm {
135 }
136
137 /// Override the effective model dtype used for multimodal tensor encoding.
138 - pub fn with_model_dtype(mut self, model_dtype: Option<ModelDtype>) -> Self {
138 + pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self {
139 self.model_dtype = model_dtype;
140 self
141 }
rust/src/chat/src/multimodal.rs
+2 -12
@@ -12,7 +12,7 @@
12 use std::collections::{HashMap, HashSet};
13 use std::fs;
14 use std::path::Path;
15 -use std::sync::{Arc, LazyLock, Once};
15 +use std::sync::{Arc, LazyLock};
16
17 use itertools::izip;
18 use llm_multimodal::{
@@ -239,7 +239,7 @@ pub(crate) async fn finalize_rendered_prompt(
239 request: &ChatRequest,
240 rendered: RenderedPrompt,
241 info: Option<&MultimodalModelInfo>,
242 - model_dtype: Option<ModelDtype>,
242 + model_dtype: ModelDtype,
243 ) -> Result<(Prompt, Option<MmFeatures>)> {
244 if !request.has_multimodal() {
245 return Ok((rendered.prompt, None));
@@ -249,16 +249,6 @@ pub(crate) async fn finalize_rendered_prompt(
249 bail_multimodal!("multimodal chat renderer must return a text prompt before expansion");
250 };
251 let media_parts = extract_media_parts(request)?;
252 - let model_dtype = model_dtype.unwrap_or_else(|| {
253 - static WARN_ONCE: Once = Once::new();
254 - WARN_ONCE.call_once(|| {
255 - warn!(
256 - "engine handshake did not report model dtype; \
257 - falling back to float32 for multimodal tensor encoding"
258 - );
259 - });
260 - ModelDtype::Float32
261 - });
252
253 let mut prompt_token_ids = info
254 .context
rust/src/engine-core-client/src/client.rs
+22 -20
@@ -290,10 +290,8 @@ impl EngineCoreClient {
290
291 // If any engine reported a dp_stats_address in its ready response, use it
292 // as the external coordinator address.
293 - let dp_stats_address: Option<String> = engines
294 - .iter()
295 - .filter_map(|e| e.ready_response.as_ref())
296 - .find_map(|r| r.dp_stats_address.clone());
293 + let dp_stats_address: Option<String> =
294 + engines.iter().find_map(|engine| engine.ready_response.dp_stats_address.clone());
295
296 let (coordinator, coordinator_output_task, coordinator_task) =
297 if let Some(coordinator_transport) = connected.coordinator {
@@ -368,40 +366,44 @@ impl EngineCoreClient {
366 /// Return the ready responses received from all engines on the input
367 /// socket.
368 pub fn ready_responses(&self) -> Vec<&EngineCoreReadyResponse> {
369 + self.engines.iter().map(|engine| &engine.ready_response).collect()
370 + }
371 +
372 + /// Return the engine-reported effective model dtype.
373 + pub fn model_dtype(&self) -> ModelDtype {
374 self.engines
372 - .iter()
373 - .filter_map(|engine| engine.ready_response.as_ref())
374 - .collect()
375 + .first()
376 + .expect("engine core client requires at least one engine")
377 + .ready_response
378 + .dtype
379 }
380
377 - /// Return the engine-reported effective model dtype, when available.
378 - pub fn model_dtype(&self) -> Option<ModelDtype> {
381 + /// Return the engine-reported Python vLLM version.
382 + pub fn vllm_version(&self) -> &str {
383 self.engines
380 - .iter()
381 - .filter_map(|engine| engine.ready_response.as_ref())
382 - .find_map(|response| response.dtype)
384 + .first()
385 + .expect("engine core client requires at least one engine")
386 + .ready_response
387 + .vllm_version
388 + .as_str()
389 }
390
391 /// Return the total number of GPU blocks summed across all connected
392 /// engines.
393 pub fn total_num_gpu_blocks(&self) -> u64 {
388 - self.engines
389 - .iter()
390 - .filter_map(|engine| engine.ready_response.as_ref())
391 - .map(|r| r.num_gpu_blocks)
392 - .sum()
394 + self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
395 }
396
397 /// Return the minimum engine-reported `max_model_len` across all engines.
398 ///
399 /// This is the auto-fitted value after KV cache profiling and may differ
400 /// from the originally configured value.
399 - pub fn max_model_len(&self) -> Option<u32> {
401 + pub fn max_model_len(&self) -> u32 {
402 self.engines
403 .iter()
402 - .filter_map(|e| e.ready_response.as_ref())
403 - .map(|r| r.max_model_len as u32)
404 + .map(|engine| engine.ready_response.max_model_len as u32)
405 .min()
406 + .expect("engine core client requires at least one engine")
407 }
408
409 /// Get the model name associated with this client used for metrics
rust/src/engine-core-client/src/client/imp.rs
+2 -1
@@ -382,6 +382,7 @@ mod tests {
382 use zeromq::{RouterSocket, Socket};
383
384 use super::*;
385 + use crate::mock_engine::default_ready_response;
386
387 async fn test_inner() -> ClientInner {
388 let mut socket = RouterSocket::new();
@@ -392,7 +393,7 @@ mod tests {
393 "test-model".to_string(),
394 &[ConnectedEngine {
395 engine_id: EngineId::from(b"engine-0"),
395 - ready_response: None,
396 + ready_response: default_ready_response(),
397 }],
398 )
399 }
rust/src/engine-core-client/src/client/state.rs
+25 -68
@@ -339,15 +339,20 @@ mod tests {
339 use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
340 use crate::EngineId;
341 use crate::client::state::EngineLoadSnapshot;
342 + use crate::mock_engine::default_ready_response;
343 use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
344 use crate::transport::ConnectedEngine;
345
346 + fn connected_engine(engine_id: EngineId) -> ConnectedEngine {
347 + ConnectedEngine {
348 + engine_id,
349 + ready_response: default_ready_response(),
350 + }
351 + }
352 +
353 #[test]
354 fn registry_rejects_duplicate_request_ids() {
347 - let mut registry = RequestRegistry::new(&[ConnectedEngine {
348 - engine_id: EngineId::from(b"engine-0"),
349 - ready_response: None,
350 - }]);
355 + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
356 registry.register("req-1".to_string(), None).unwrap();
357 let error = registry.register("req-1".to_string(), None).unwrap_err();
358 assert!(matches!(
@@ -358,10 +363,7 @@ mod tests {
363
364 #[test]
365 fn registry_removes_finished_request_on_output() {
361 - let mut registry = RequestRegistry::new(&[ConnectedEngine {
362 - engine_id: EngineId::from(b"engine-0"),
363 - ready_response: None,
364 - }]);
366 + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
367 registry.register("req-1".to_string(), None).unwrap();
368
369 let sender = registry.sender_for_output(&EngineCoreOutput {
@@ -376,10 +378,7 @@ mod tests {
378
379 #[test]
380 fn registry_closes_all_requests_on_failure() {
379 - let mut registry = RequestRegistry::new(&[ConnectedEngine {
380 - engine_id: EngineId::from(b"engine-0"),
381 - ready_response: None,
382 - }]);
381 + let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
382 registry.register("req-1".to_string(), None).unwrap();
383 registry.register("req-2".to_string(), None).unwrap();
384
@@ -394,14 +393,8 @@ mod tests {
393 let engine_0 = EngineId::from_engine_index(0);
394 let engine_1 = EngineId::from_engine_index(1);
395 let mut registry = RequestRegistry::new(&[
397 - ConnectedEngine {
398 - engine_id: engine_0.clone(),
399 - ready_response: None,
400 - },
401 - ConnectedEngine {
402 - engine_id: engine_1.clone(),
403 - ready_response: None,
404 - },
396 + connected_engine(engine_0.clone()),
397 + connected_engine(engine_1.clone()),
398 ]);
399 let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
400 let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
@@ -428,14 +421,8 @@ mod tests {
421 let engine_0 = EngineId::from_engine_index(0);
422 let engine_1 = EngineId::from_engine_index(1);
423 let mut registry = RequestRegistry::new(&[
431 - ConnectedEngine {
432 - engine_id: engine_0.clone(),
433 - ready_response: None,
434 - },
435 - ConnectedEngine {
436 - engine_id: engine_1.clone(),
437 - ready_response: None,
438 - },
424 + connected_engine(engine_0.clone()),
425 + connected_engine(engine_1.clone()),
426 ]);
427
428 let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
@@ -488,14 +475,8 @@ mod tests {
475 let engine_0 = EngineId::from_engine_index(0);
476 let engine_1 = EngineId::from_engine_index(1);
477 let mut registry = RequestRegistry::new(&[
491 - ConnectedEngine {
492 - engine_id: engine_0.clone(),
493 - ready_response: None,
494 - },
495 - ConnectedEngine {
496 - engine_id: engine_1.clone(),
497 - ready_response: None,
498 - },
478 + connected_engine(engine_0.clone()),
479 + connected_engine(engine_1.clone()),
480 ]);
481
482 assert!(registry.apply_scheduler_counts(
@@ -523,18 +504,9 @@ mod tests {
504 let engine_1 = EngineId::from_engine_index(1);
505 let engine_2 = EngineId::from_engine_index(2);
506 let mut registry = RequestRegistry::new(&[
526 - ConnectedEngine {
527 - engine_id: engine_0.clone(),
528 - ready_response: None,
529 - },
530 - ConnectedEngine {
531 - engine_id: engine_1.clone(),
532 - ready_response: None,
533 - },
534 - ConnectedEngine {
535 - engine_id: engine_2.clone(),
536 - ready_response: None,
537 - },
507 + connected_engine(engine_0.clone()),
508 + connected_engine(engine_1.clone()),
509 + connected_engine(engine_2.clone()),
510 ]);
511
512 // Explicitly target rank 2 (third engine).
@@ -555,14 +527,8 @@ mod tests {
527 let engine_0 = EngineId::from_engine_index(0);
528 let engine_1 = EngineId::from_engine_index(1);
529 let mut registry = RequestRegistry::new(&[
558 - ConnectedEngine {
559 - engine_id: engine_0.clone(),
560 - ready_response: None,
561 - },
562 - ConnectedEngine {
563 - engine_id: engine_1.clone(),
564 - ready_response: None,
565 - },
530 + connected_engine(engine_0.clone()),
531 + connected_engine(engine_1.clone()),
532 ]);
533
534 // Load-balance: first two go to engine_0 and engine_1.
@@ -577,14 +543,8 @@ mod tests {
543 #[test]
544 fn register_with_out_of_range_rank_returns_error() {
545 let mut registry = RequestRegistry::new(&[
580 - ConnectedEngine {
581 - engine_id: EngineId::from_engine_index(0),
582 - ready_response: None,
583 - },
584 - ConnectedEngine {
585 - engine_id: EngineId::from_engine_index(1),
586 - ready_response: None,
587 - },
546 + connected_engine(EngineId::from_engine_index(0)),
547 + connected_engine(EngineId::from_engine_index(1)),
548 ]);
549
550 let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
@@ -600,10 +560,7 @@ mod tests {
560 #[test]
561 fn register_with_rank_on_single_engine_only_accepts_zero() {
562 let engine_0 = EngineId::from_engine_index(0);
603 - let mut registry = RequestRegistry::new(&[ConnectedEngine {
604 - engine_id: engine_0.clone(),
605 - ready_response: None,
606 - }]);
563 + let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
564
565 let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
566 assert_eq!(chosen, engine_0);
rust/src/engine-core-client/src/mock_engine.rs
+2 -1
@@ -47,7 +47,8 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
47 max_model_len: DEFAULT_MOCK_MAX_MODEL_LEN,
48 num_gpu_blocks: DEFAULT_MOCK_NUM_GPU_BLOCKS,
49 dp_stats_address: None,
50 - dtype: Some(ModelDtype::Float32),
50 + dtype: ModelDtype::Float32,
51 + vllm_version: "test-vllm-version".to_string(),
52 }
53 }
54
rust/src/engine-core-client/src/protocol/handshake.rs
+3 -23
@@ -39,10 +39,9 @@ pub struct EngineCoreReadyResponse {
39 /// DP coordinator stats publish address, if applicable.
40 pub dp_stats_address: Option<String>,
41 /// Effective model dtype after Python vLLM resolves `--dtype`.
42 - // TODO: This is currently not wired up on the engine side. After it's added, remove `Option`
43 - // and `serde(default)`.
44 - #[serde(default)]
45 - pub dtype: Option<ModelDtype>,
42 + pub dtype: ModelDtype,
43 + /// Python vLLM version reported by the engine process.
44 + pub vllm_version: String,
45 }
46
47 /// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -69,22 +68,3 @@ pub struct HandshakeInitMessage {
68 pub addresses: HandshakeAddresses,
69 pub parallel_config: BTreeMap<String, OpaqueValue>,
70 }
72 -
73 -#[cfg(test)]
74 -mod tests {
75 - use super::EngineCoreReadyResponse;
76 - use crate::protocol::ModelDtype;
77 -
78 - #[test]
79 - fn ready_response_accepts_effective_dtype() {
80 - let response: EngineCoreReadyResponse = serde_json::from_value(serde_json::json!({
81 - "max_model_len": 4096,
82 - "num_gpu_blocks": 2,
83 - "dp_stats_address": null,
84 - "dtype": "bfloat16"
85 - }))
86 - .unwrap();
87 -
88 - assert_eq!(response.dtype, Some(ModelDtype::BFloat16));
89 - }
90 -}
rust/src/engine-core-client/src/test_utils.rs
+3 -8
@@ -10,9 +10,9 @@ use crate::EngineId;
10 pub use crate::mock_engine::{MockCoordinatorSockets, MockEngineSockets};
11 use crate::mock_engine::{
12 MockEngineConfig, MockEngineDataSockets, connect_to_bootstrapped_frontend, connect_to_frontend,
13 + default_ready_response,
14 };
14 -use crate::protocol::ModelDtype;
15 -use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage};
15 +use crate::protocol::handshake::HandshakeInitMessage;
16
17 /// Per-test IPC endpoint namespace backed by a unique temporary directory.
18 ///
@@ -57,12 +57,7 @@ fn test_mock_engine_config() -> MockEngineConfig {
57 MockEngineConfig {
58 local: true,
59 headless: true,
60 - ready_response: EngineCoreReadyResponse {
61 - max_model_len: 4096,
62 - num_gpu_blocks: 0,
63 - dp_stats_address: None,
64 - dtype: Some(ModelDtype::Float32),
65 - },
60 + ready_response: default_ready_response(),
61 ..Default::default()
62 }
63 }
rust/src/engine-core-client/src/tests/client.rs
+1
@@ -925,6 +925,7 @@ async fn client_fail_closes_when_main_output_path_receives_dp_control() {
925 .await;
926 assert_eq!(client.engine_identities()[0], b"engine-0");
927 assert!(client.ready_responses()[0].max_model_len > 0);
928 + assert_eq!(client.vllm_version(), "test-vllm-version");
929
930 let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
931 let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
rust/src/engine-core-client/src/transport.rs
+40 -50
@@ -104,8 +104,8 @@ pub struct ConnectedEngine {
104 /// The identity of the connected engine.
105 pub engine_id: EngineId,
106 /// Post-initialization configuration received from the engine on the input
107 - /// socket registration message. `None` until the registration is received.
108 - pub ready_response: Option<EngineCoreReadyResponse>,
107 + /// socket registration message.
108 + pub ready_response: EngineCoreReadyResponse,
109 }
110
111 /// Represents the connected shared transport plus all registered engines after
@@ -295,18 +295,9 @@ pub async fn connect_handshake(
295 }
296 }
297
298 - // 4. Wait for every engine to connect to the shared input socket and register itself. The
299 - // `ready_response` is a placeholder; it is populated for each engine by
300 - // `wait_for_input_registrations` below.
301 - let mut engines: Vec<_> = engines
302 - .into_keys()
303 - .map(|engine_id| ConnectedEngine {
304 - engine_id,
305 - ready_response: None,
306 - })
307 - .collect();
308 -
309 - wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
298 + // 6. Wait for every engine to connect to the shared input socket and register itself.
299 + let engines =
300 + wait_for_input_registrations(&mut input_socket, engines.into_keys(), ready_timeout).await?;
301 debug!(
302 engine_count = engines.len(),
303 "all engines registered on shared input socket"
@@ -349,15 +340,13 @@ pub async fn connect_bootstrapped(
340 let mut output_socket = PullSocket::new();
341 let output_address = output_socket.bind(output_address).await?.to_string();
342
352 - // TODO: follow start rank
353 - let mut engines = (0..engine_count)
354 - .map(|index| ConnectedEngine {
355 - engine_id: EngineId::from((index as u16).to_le_bytes().to_vec()),
356 - ready_response: None,
357 - })
358 - .collect::<Vec<_>>();
359 -
360 - wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
343 + let engines = wait_for_input_registrations(
344 + &mut input_socket,
345 + // TODO: follow start rank
346 + (0..engine_count).map(|index| EngineId::from((index as u16).to_le_bytes().to_vec())),
347 + ready_timeout,
348 + )
349 + .await?;
350 info!(
351 engine_count = engines.len(),
352 "bootstrapped engines connected"
@@ -455,17 +444,14 @@ async fn send_init_message(
444 /// Simplify API server handshake"), the payload is a msgpack-encoded
445 /// [`EngineCoreReadyResponse`] carrying post-initialization values such as
446 /// `max_model_len`.
458 -///
459 -/// Older engines sent an empty second frame here just to establish the
460 -/// ROUTER/DEALER backchannel, with no structured payload on the input socket.
461 -/// We continue to tolerate that legacy shape so the frontend can still connect
462 -/// to slightly older local engine checkouts.
447 async fn wait_for_input_registrations(
448 input_socket: &mut RouterSocket,
465 - engines: &mut [ConnectedEngine],
449 + expected_engines: impl IntoIterator<Item = EngineId>,
450 ready_timeout: Duration,
467 -) -> Result<()> {
468 - let mut pending = engines.iter().map(|e| e.engine_id.clone()).collect::<BTreeSet<_>>();
451 +) -> Result<Vec<ConnectedEngine>> {
452 + let expected_engines = expected_engines.into_iter().collect::<Vec<_>>();
453 + let mut pending = expected_engines.iter().cloned().collect::<BTreeSet<_>>();
454 + let mut ready_responses = BTreeMap::new();
455
456 while !pending.is_empty() {
457 let registration = timeout(ready_timeout, input_socket.recv()).await.map_err(|_| {
@@ -489,29 +475,33 @@ async fn wait_for_input_registrations(
475 );
476 }
477
492 - let ready_response = if frames[1].is_empty() {
493 - debug!(
494 - ?actual_id,
495 - "received legacy empty input registration from engine"
496 - );
497 - None
498 - } else {
499 - let ready_response: EngineCoreReadyResponse = decode_msgpack(&frames[1])?;
500 - debug!(
501 - ?actual_id,
502 - ?ready_response,
503 - "received input registration from engine"
478 + if frames[1].is_empty() {
479 + bail_unexpected_handshake_message!(
480 + "expected msgpack EngineCoreReadyResponse for engine input registration, got empty payload from engine id {actual_id:?}"
481 );
505 - Some(ready_response)
506 - };
507 -
508 - // Store the ready response in the corresponding engine entry.
509 - if let Some(engine) = engines.iter_mut().find(|e| e.engine_id == actual_id) {
510 - engine.ready_response = ready_response;
482 }
483 +
484 + let ready_response: EngineCoreReadyResponse = decode_msgpack(&frames[1])?;
485 + debug!(
486 + ?actual_id,
487 + ?ready_response,
488 + "received input registration from engine"
489 + );
490 + ready_responses.insert(actual_id, ready_response);
491 }
492
514 - Ok(())
493 + Ok(expected_engines
494 + .into_iter()
495 + .map(|engine_id| {
496 + let ready_response = ready_responses
497 + .remove(&engine_id)
498 + .expect("every expected engine id has a decoded ready response");
499 + ConnectedEngine {
500 + engine_id,
501 + ready_response,
502 + }
503 + })
504 + .collect())
505 }
506
507 /// Send an encoded message to the engine through the input socket.
rust/src/mock-engine/src/tests.rs
+2 -1
@@ -98,7 +98,8 @@ async fn mock_engine_connects_over_tcp() {
98 let (client, shutdown, task) = connect_with_mock(handshake_address, 1, 1).await;
99 assert_eq!(client.engine_count(), 1);
100 assert_eq!(client.engine_identities()[0], &[0, 0]);
101 - assert_eq!(client.max_model_len(), Some(1024 * 1024));
101 + assert_eq!(client.max_model_len(), 1024 * 1024);
102 + assert_eq!(client.vllm_version(), "test-vllm-version");
103 shutdown_mock(client, shutdown, task).await;
104 }
105
rust/src/server/src/routes.rs
+2
@@ -6,6 +6,7 @@ mod load;
6 mod metrics;
7 pub(crate) mod openai;
8 mod sleep;
9 +mod version;
10
11 use std::sync::Arc;
12
@@ -35,6 +36,7 @@ fn build_router_with_dev_mode(state: Arc<AppState>, dev_mode_enabled: bool) -> R
36 .route("/health", get(health::health))
37 .route("/metrics", get(metrics::scrape))
38 .route("/load", get(load::load))
39 + .route("/version", get(version::version))
40 // OpenAI-compatible endpoints
41 .route("/v1/models", get(openai::list_models))
42 .route("/v1/completions", post(openai::completions))
rust/src/server/src/routes/tests.rs
+21
@@ -994,6 +994,27 @@ async fn list_models_returns_configured_model() {
994 assert_eq!(json["data"][0]["id"], "Qwen/Qwen1.5-0.5B-Chat");
995 }
996
997 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
998 +#[serial]
999 +async fn version_returns_engine_vllm_version() {
1000 + let mut app = test_app().await;
1001 + let response = app
1002 + .call(Request::builder().uri("/version").body(Body::empty()).expect("build request"))
1003 + .await
1004 + .expect("call app");
1005 +
1006 + assert_eq!(response.status(), StatusCode::OK);
1007 + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
1008 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1009 + assert_eq!(
1010 + json,
1011 + json!({
1012 + "version": "test-vllm-version",
1013 + "rust_frontend_version": env!("CARGO_PKG_VERSION"),
1014 + })
1015 + );
1016 +}
1017 +
1018 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1019 #[serial]
1020 async fn http_metrics_record_list_models_requests() {
rust/src/server/src/routes/version.rs new
+23
@@ -0,0 +1,23 @@
1 +use std::sync::Arc;
2 +
3 +use axum::Json;
4 +use axum::extract::State;
5 +use serde::Serialize;
6 +
7 +use crate::state::AppState;
8 +
9 +#[derive(Serialize)]
10 +pub(crate) struct VersionResponse {
11 + version: String,
12 + rust_frontend_version: &'static str,
13 +}
14 +
15 +/// Get engine and Rust frontend version metadata.
16 +pub async fn version(State(state): State<Arc<AppState>>) -> Json<VersionResponse> {
17 + let version = state.engine_core_client().vllm_version().to_string();
18 +
19 + Json(VersionResponse {
20 + version,
21 + rust_frontend_version: env!("CARGO_PKG_VERSION"),
22 + })
23 +}
rust/src/text/src/lib.rs
+5 -7
@@ -45,9 +45,9 @@ pub struct TextLlm {
45 /// Tokenizer/model metadata backend responsible for prompt encode/decode
46 /// and sampling hints.
47 backend: DynTextBackend,
48 - /// Context window size derived by the backend or from engine startup
49 - /// handshake, with optional override from config.
50 - max_model_len: Option<u32>,
48 + /// Context window size reported by the engine startup handshake, with
49 + /// optional override from config.
50 + max_model_len: u32,
51 }
52
53 impl TextLlm {
@@ -71,7 +71,7 @@ impl TextLlm {
71 /// This takes priority over both the engine-reported default and any
72 /// tokenizer/model metadata exposed by the backend.
73 pub fn with_max_model_len(mut self, max_model_len: u32) -> Self {
74 - self.max_model_len = Some(max_model_len);
74 + self.max_model_len = max_model_len;
75 self
76 }
77
@@ -129,9 +129,7 @@ impl TextLlm {
129 };
130
131 let mut sampling_hints = self.backend.sampling_hints()?;
132 - if let Some(max_model_len) = self.max_model_len {
133 - sampling_hints.max_model_len = Some(max_model_len);
134 - }
132 + sampling_hints.max_model_len = Some(self.max_model_len);
133 let PreparedTextRequest {
134 text_request,
135 generate_request,
vllm/v1/engine/__init__.py
+2 -1
@@ -75,7 +75,8 @@ class EngineCoreReadyResponse:
75 max_model_len: int
76 num_gpu_blocks: int
77 dp_stats_address: str | None
78 - dtype: str | None = None
78 + dtype: str
79 + vllm_version: str
80
81
82 class EngineCoreRequest(
vllm/v1/engine/core.py
+1
@@ -1464,6 +1464,7 @@ class EngineCoreProc(EngineCore):
1464 num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
1465 dp_stats_address=self.frontend_stats_publish_address,
1466 dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
1467 + vllm_version=VLLM_VERSION,
1468 )
1469 ready_payload = msgspec.msgpack.encode(ready_response)
1470 for input_socket in input_sockets: