A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][Rust] Sync EngineCoreReadyResponse with the Python dataclass (#45557)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Will Eaton <weaton@redhat.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Will Eaton committed
Jun 15, 2026 at 02:49 UTC
ddad5dbda20c6eee443a239790e359a22cb5d65e
4 files changed
+54
-1
rust/src/engine-core-client/src/mock_engine.rs
+5
@@ -15,6 +15,8 @@ use crate::protocol::{ModelDtype, decode_msgpack, encode_msgpack};
15
pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;
16
/// Default KV block count advertised by reusable mock engine helpers.
17
pub const DEFAULT_MOCK_NUM_GPU_BLOCKS: u64 = 0;
18
+/// Default KV block size (tokens per block)
19
+pub const DEFAULT_MOCK_BLOCK_SIZE: u64 = 16;
20
21
/// Startup behavior for one mock engine joining a frontend.
22
#[derive(Debug, Clone)]
@@ -46,9 +48,12 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
48
EngineCoreReadyResponse {
49
max_model_len: DEFAULT_MOCK_MAX_MODEL_LEN,
50
num_gpu_blocks: DEFAULT_MOCK_NUM_GPU_BLOCKS,
51
+ block_size: DEFAULT_MOCK_BLOCK_SIZE,
52
dp_stats_address: None,
53
dtype: ModelDtype::Float32,
54
vllm_version: "test-vllm-version".to_string(),
55
+ kv_cache_size_tokens: None,
56
+ kv_cache_max_concurrency: None,
57
}
58
}
59
rust/src/engine-core-client/src/protocol/handshake.rs
+7
-1
@@ -28,7 +28,7 @@ pub struct ReadyMessage {
28
/// profiling).
29
///
30
/// Original Python definition:
31
-/// <https://github.com/vllm-project/vllm/blob/c8d98f81f6/vllm/v1/engine/__init__.py#L67-L77>
31
+/// <https://github.com/vllm-project/vllm/blob/c9340e6f35/vllm/v1/engine/__init__.py#L68-L80>
32
#[derive(Debug, Clone, Serialize, Deserialize)]
33
pub struct EngineCoreReadyResponse {
34
/// Engine-reported maximum model context length (auto-fitted after
@@ -36,12 +36,18 @@ pub struct EngineCoreReadyResponse {
36
pub max_model_len: u64,
37
/// Number of GPU blocks available for KV cache on this engine.
38
pub num_gpu_blocks: u64,
39
+ /// KV cache block size (tokens per block).
40
+ pub block_size: u64,
41
/// DP coordinator stats publish address, if applicable.
42
pub dp_stats_address: Option<String>,
43
/// Effective model dtype after Python vLLM resolves `--dtype`.
44
pub dtype: ModelDtype,
45
/// Python vLLM version reported by the engine process.
46
pub vllm_version: String,
47
+ /// Total KV cache capacity in tokens, if reported.
48
+ pub kv_cache_size_tokens: Option<u64>,
49
+ /// Maximum achievable request concurrency given the KV cache, if reported.
50
+ pub kv_cache_max_concurrency: Option<f64>,
51
}
52
53
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
rust/src/engine-core-client/src/tests/client.rs
+18
@@ -2445,6 +2445,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2445
let inline_prompt_frames = lines.next().expect("missing inline prompt logprobs fixture line");
2446
let multipart_prompt_frames =
2447
lines.next().expect("missing multipart prompt logprobs fixture line");
2448
+ let ready_response_hex = lines.next().expect("missing ready response fixture line");
2449
2450
let request_bytes = hex::decode(request_hex).unwrap();
2451
let multimodal_request_bytes = hex::decode(multimodal_request_hex).unwrap();
@@ -2554,6 +2555,23 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2555
.as_ref()
2556
.expect("multipart prompt logprobs decoded"),
2557
);
2558
+
2559
+ let map_keys = |bytes: &[u8]| -> BTreeSet<String> {
2560
+ match decode_value(bytes) {
2561
+ Value::Map(entries) => entries
2562
+ .into_iter()
2563
+ .filter_map(|(key, _)| key.as_str().map(str::to_owned))
2564
+ .collect(),
2565
+ other => panic!("ready response should encode as a map, got {other:?}"),
2566
+ }
2567
+ };
2568
+ let python_ready_keys = map_keys(&hex::decode(ready_response_hex).unwrap());
2569
+ let rust_ready_keys =
2570
+ map_keys(&rmp_serde::to_vec_named(&crate::mock_engine::default_ready_response()).unwrap());
2571
+ assert_eq!(
2572
+ rust_ready_keys, python_ready_keys,
2573
+ "EngineCoreReadyResponse drifted from the Python dataclass",
2574
+ );
2575
}
2576
2577
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
rust/src/engine-core-client/src/tests/python_compat.py
+24
@@ -10,6 +10,7 @@
10
# ]
11
# ///
12
13
+from dataclasses import dataclass
14
from enum import Enum, IntEnum
15
16
import msgpack
@@ -337,6 +338,28 @@ multipart_prompt_logprobs = engine_outputs_wire(
338
)
339
)
340
341
+
342
+@dataclass
343
+class EngineCoreReadyResponse:
344
+ max_model_len: int
345
+ num_gpu_blocks: int
346
+ block_size: int
347
+ dp_stats_address: str | None
348
+ dtype: str
349
+ vllm_version: str
350
+ kv_cache_size_tokens: int | None = None
351
+ kv_cache_max_concurrency: float | None = None
352
+
353
+
354
+ready_response = EngineCoreReadyResponse(
355
+ max_model_len=32768,
356
+ num_gpu_blocks=1000,
357
+ block_size=16,
358
+ dp_stats_address=None,
359
+ dtype="float32",
360
+ vllm_version="0.0.0",
361
+)
362
+
363
print(msgspec.msgpack.encode(request).hex())
364
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
365
print(msgspec.msgpack.encode(outputs).hex())
@@ -354,3 +377,4 @@ print(
377
for frame in encode_output_frames(multipart_prompt_logprobs, size_threshold=1)
378
)
379
)
380
+print(msgspec.msgpack.encode(ready_response).hex())