@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 25fcb65d51

[Rust Frontend] Use enum-backed domain types for engine outputs and structured outputs (#47283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jul 2, 2026 at 17:41 UTC 25fcb65d51deef0026aa34e6067703da4a91f956
16 files changed +760 -603
rust/src/chat/src/output/default/structural_tag.rs
+12 -18
@@ -47,9 +47,8 @@ pub(super) fn apply_structural_tag_constraint(
47
48 // Overwrite any existing structured output settings with the structural tag constraint.
49 request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
50 - structural_tag: Some(structural_tag),
50 backend: StructuredOutputBackend::Xgrammar,
52 - ..Default::default()
51 + ..StructuredOutputsParams::structural_tag(structural_tag)
52 });
53
54 Ok(())
@@ -119,10 +118,11 @@ mod tests {
118 .as_ref()
119 .expect("structured outputs should be set");
120 assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
122 - serde_json::from_str(
123 - params.structural_tag.as_deref().expect("structural_tag should be set"),
124 - )
125 - .expect("structural_tag should be valid JSON")
121 + let structural_tag = params
122 + .constraint
123 + .as_structural_tag()
124 + .expect("structured output constraint should be structural_tag");
125 + serde_json::from_str(structural_tag).expect("structural_tag should be valid JSON")
126 }
127
128 fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams {
@@ -161,9 +161,8 @@ mod tests {
161 fn auto_strict_tool_choice_overwrites_existing_json_guidance() {
162 let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
163 request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
164 - json: Some(json!({"type": "object"})),
164 backend: StructuredOutputBackend::Xgrammar,
166 - ..Default::default()
165 + ..StructuredOutputsParams::json(json!({"type": "object"}))
166 });
167 let parser = qwen3_coder_parser(&request.tools);
168
@@ -171,8 +170,7 @@ mod tests {
170 .expect("structural tag should build");
171
172 let params = structured_outputs(&request);
174 - assert!(params.json.is_none());
175 - assert!(params.structural_tag.is_some());
173 + assert!(params.constraint.is_structural_tag());
174 let tag = structural_tag_value(&request);
175 assert_eq!(tag["type"], "structural_tag");
176 assert!(tag.to_string().contains("search"));
@@ -195,9 +193,8 @@ mod tests {
193 fn required_tool_choice_overwrites_existing_json_object_guidance() {
194 let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
195 request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
198 - json_object: Some(true),
196 backend: StructuredOutputBackend::Xgrammar,
200 - ..Default::default()
197 + ..StructuredOutputsParams::json_object()
198 });
199 let parser = qwen3_coder_parser(&request.tools);
200
@@ -205,8 +202,7 @@ mod tests {
202 .expect("structural tag should build");
203
204 let params = structured_outputs(&request);
208 - assert!(params.json_object.is_none());
209 - assert!(params.structural_tag.is_some());
205 + assert!(params.constraint.is_structural_tag());
206 let tag = structural_tag_value(&request);
207 assert_eq!(tag["type"], "structural_tag");
208 assert!(tag.to_string().contains("search"));
@@ -245,9 +241,8 @@ mod tests {
241 fn none_tool_choice_preserves_existing_json_object_guidance() {
242 let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
243 request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
248 - json_object: Some(true),
244 backend: StructuredOutputBackend::Xgrammar,
250 - ..Default::default()
245 + ..StructuredOutputsParams::json_object()
246 });
247 let parser = qwen3_coder_parser(&request.tools);
248
@@ -255,7 +250,6 @@ mod tests {
250 .expect("structural tag decision should succeed");
251
252 let params = structured_outputs(&request);
258 - assert_eq!(params.json_object, Some(true));
259 - assert!(params.structural_tag.is_none());
253 + assert!(params.constraint.is_json_object());
254 }
255 }
rust/src/chat/tests/chat.rs
+34 -23
@@ -16,7 +16,7 @@ use vllm_engine_core_client::protocol::logprobs::{
16 Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
17 };
18 use vllm_engine_core_client::protocol::output::{
19 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, StopReason,
19 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs, StopReason,
20 };
21 use vllm_engine_core_client::protocol::request::EngineCoreRequest;
22 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
@@ -342,7 +342,7 @@ async fn chat_streams_text_events() {
342 );
343 send_outputs(
344 push,
345 - EngineCoreOutputs {
345 + RequestBatchOutputs {
346 outputs: vec![
347 request_output("chat-1", vec![b'H' as u32], None, None),
348 request_output(
@@ -354,7 +354,8 @@ async fn chat_streams_text_events() {
354 ],
355 finished_requests: Some(BTreeSet::from(["chat-1".to_string()])),
356 ..Default::default()
357 - },
357 + }
358 + .into(),
359 )
360 .await;
361 })
@@ -454,7 +455,7 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
455 let _ = recv_engine_message(dealer).await;
456 send_outputs(
457 push,
457 - EngineCoreOutputs {
458 + RequestBatchOutputs {
459 outputs: vec![
460 request_output("chat-utf8", bytes_to_token_ids(&[0xe4]), None, None),
461 request_output(
@@ -466,7 +467,8 @@ async fn chat_stream_waits_for_complete_utf8_before_emitting() {
467 ],
468 finished_requests: Some(BTreeSet::from(["chat-utf8".to_string()])),
469 ..Default::default()
469 - },
470 + }
471 + .into(),
472 )
473 .await;
474 })
@@ -542,7 +544,7 @@ async fn chat_stream_flushes_held_text_on_finish() {
544 let _ = recv_engine_message(dealer).await;
545 send_outputs(
546 push,
545 - EngineCoreOutputs {
547 + RequestBatchOutputs {
548 outputs: vec![request_output(
549 "chat-final-flush",
550 bytes_to_token_ids(b"ok st"),
@@ -551,7 +553,8 @@ async fn chat_stream_flushes_held_text_on_finish() {
553 )],
554 finished_requests: Some(BTreeSet::from(["chat-final-flush".to_string()])),
555 ..Default::default()
554 - },
556 + }
557 + .into(),
558 )
559 .await;
560 })
@@ -662,7 +665,7 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
665 let _ = recv_engine_message(dealer).await;
666 send_outputs(
667 push,
665 - EngineCoreOutputs {
668 + RequestBatchOutputs {
669 outputs: vec![request_output(
670 "chat-4",
671 vec![UNKNOWN_DECODE_TOKEN_ID],
@@ -670,7 +673,8 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
673 None,
674 )],
675 ..Default::default()
673 - },
676 + }
677 + .into(),
678 )
679 .await;
680 })
@@ -725,7 +729,7 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
729 let _ = recv_engine_message(dealer).await;
730 send_outputs(
731 push,
728 - EngineCoreOutputs {
732 + RequestBatchOutputs {
733 outputs: vec![request_output(
734 "chat-include-stop",
735 vec![b'H' as u32, b'i' as u32, b'!' as u32],
@@ -734,7 +738,8 @@ async fn chat_stream_preserves_terminal_stop_token_when_requested() {
738 )],
739 finished_requests: Some(BTreeSet::from(["chat-include-stop".to_string()])),
740 ..Default::default()
737 - },
741 + }
742 + .into(),
743 )
744 .await;
745 })
@@ -812,7 +817,7 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
817 let _ = recv_engine_message(dealer).await;
818 send_outputs(
819 push,
815 - EngineCoreOutputs {
820 + RequestBatchOutputs {
821 outputs: vec![
822 request_output(
823 "chat-reasoning",
@@ -841,7 +846,8 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
846 ],
847 finished_requests: Some(BTreeSet::from(["chat-reasoning".to_string()])),
848 ..Default::default()
844 - },
849 + }
850 + .into(),
851 )
852 .await;
853 })
@@ -955,7 +961,7 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
961 let _ = recv_engine_message(dealer).await;
962 send_outputs(
963 push,
958 - EngineCoreOutputs {
964 + RequestBatchOutputs {
965 outputs: vec![request_output(
966 "chat-collect",
967 bytes_to_token_ids(b"<think>inner</think>outer"),
@@ -964,7 +970,8 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
970 )],
971 finished_requests: Some(BTreeSet::from(["chat-collect".to_string()])),
972 ..Default::default()
967 - },
973 + }
974 + .into(),
975 )
976 .await;
977 })
@@ -1018,7 +1025,7 @@ async fn chat_explicitly_disables_reasoning_parser() {
1025 let _ = recv_engine_message(dealer).await;
1026 send_outputs(
1027 push,
1021 - EngineCoreOutputs {
1028 + RequestBatchOutputs {
1029 outputs: vec![
1030 request_output(
1031 "chat-reasoning-disabled",
@@ -1049,7 +1056,8 @@ async fn chat_explicitly_disables_reasoning_parser() {
1056 "chat-reasoning-disabled".to_string()
1057 ])),
1058 ..Default::default()
1052 - },
1059 + }
1060 + .into(),
1061 )
1062 .await;
1063 })
@@ -1096,7 +1104,7 @@ async fn chat_stream_parses_tool_calls_automatically() {
1104 let _ = recv_engine_message(dealer).await;
1105 send_outputs(
1106 push,
1099 - EngineCoreOutputs {
1107 + RequestBatchOutputs {
1108 outputs: vec![
1109 request_output(
1110 "chat-tool",
@@ -1121,7 +1129,8 @@ async fn chat_stream_parses_tool_calls_automatically() {
1129 ],
1130 finished_requests: Some(BTreeSet::from(["chat-tool".to_string()])),
1131 ..Default::default()
1124 - },
1132 + }
1133 + .into(),
1134 )
1135 .await;
1136 })
@@ -1204,7 +1213,7 @@ async fn chat_collect_message_preserves_tool_call_arguments_in_final_only_mode()
1213 let _ = recv_engine_message(dealer).await;
1214 send_outputs(
1215 push,
1207 - EngineCoreOutputs {
1216 + RequestBatchOutputs {
1217 outputs: vec![
1218 request_output(
1219 "chat-final-only-tool",
@@ -1231,7 +1240,8 @@ async fn chat_collect_message_preserves_tool_call_arguments_in_final_only_mode()
1240 "chat-final-only-tool".to_string()
1241 ])),
1242 ..Default::default()
1234 - },
1243 + }
1244 + .into(),
1245 )
1246 .await;
1247 })
@@ -1282,7 +1292,7 @@ async fn chat_stream_and_collect_preserve_prompt_and_sample_logprobs() {
1292 let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
1293 send_outputs(
1294 push,
1285 - EngineCoreOutputs {
1295 + RequestBatchOutputs {
1296 outputs: vec![
1297 request_output_with_logprobs(
1298 &request.request_id,
@@ -1303,7 +1313,8 @@ async fn chat_stream_and_collect_preserve_prompt_and_sample_logprobs() {
1313 ],
1314 finished_requests: Some(BTreeSet::from([request.request_id])),
1315 ..Default::default()
1306 - },
1316 + }
1317 + .into(),
1318 )
1319 .await;
1320 }
rust/src/engine-core-client/src/client/imp.rs
+5 -6
@@ -17,7 +17,7 @@ use crate::client::{AbortCause, AbortRequest};
17 use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
18 use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
19 use crate::protocol::encode_msgpack;
20 -use crate::protocol::output::{ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs};
20 +use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
21 use crate::protocol::request::EngineCoreRequestType;
22 use crate::protocol::stats::SchedulerStats;
23 use crate::protocol::utility::UtilityOutput;
@@ -351,8 +351,8 @@ pub(crate) async fn run_output_dispatcher_loop(
351 )),
352 }?;
353
354 - match outputs.classify() {
355 - ClassifiedEngineCoreOutputs::RequestBatch(batch) => {
354 + match outputs {
355 + EngineCoreOutputs::RequestBatch(batch) => {
356 let senders = inner.take_senders_for_outputs(&batch.outputs);
357 for (output, sender) in batch.outputs.into_iter().zip(senders) {
358 let request_id = output.request_id.clone();
@@ -403,7 +403,7 @@ pub(crate) async fn run_output_dispatcher_loop(
403 let (running, waiting) = inner.lora_adapter_states();
404 lora_info.update(&METRICS.scheduler, running, waiting);
405 }
406 - ClassifiedEngineCoreOutputs::Utility(utility) => {
406 + EngineCoreOutputs::Utility(utility) => {
407 let call_id = utility.output.call_id;
408 if inner.resolve_utility_output(utility.output) {
409 trace!(
@@ -419,8 +419,7 @@ pub(crate) async fn run_output_dispatcher_loop(
419 );
420 }
421 }
422 - other @ (ClassifiedEngineCoreOutputs::DpControl { .. }
423 - | ClassifiedEngineCoreOutputs::Other(_)) => {
422 + other => {
423 Err::<(), _>(unexpected_dispatcher_output!(
424 "received unexpected output on main dispatcher path: {other:?}"
425 ))?;
rust/src/engine-core-client/src/coordinator/inproc.rs
+5 -5
@@ -11,7 +11,7 @@ use crate::client::imp::ClientInner;
11 use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
12 use crate::error::{Error, Result, bail_unexpected_coordinator_output};
13 use crate::protocol::encode_msgpack;
14 -use crate::protocol::output::{ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs};
14 +use crate::protocol::output::{DpControlMessage, DpControlOutput, EngineCoreOutputs};
15 use crate::protocol::request::EngineCoreRequestType;
16
17 /// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
@@ -109,19 +109,19 @@ impl InProcCoordinatorRunner {
109 /// Apply one engine-originated control output to the coordinator state
110 /// machine.
111 async fn handle_outputs(&mut self, outputs: EngineCoreOutputs) -> Result<()> {
112 - match outputs.classify() {
113 - ClassifiedEngineCoreOutputs::RequestBatch(batch)
112 + match outputs {
113 + EngineCoreOutputs::RequestBatch(batch)
114 if batch.outputs.is_empty() && batch.finished_requests.is_none() =>
115 {
116 // Stats-only output for coordinator.
117 // Ignore since the Rust coordinator doesn't track stats for
118 // routing decisions.
119 }
120 - ClassifiedEngineCoreOutputs::DpControl {
120 + EngineCoreOutputs::DpControl(DpControlOutput {
121 engine_index,
122 control,
123 ..
124 - } => match control {
124 + }) => match control {
125 // The engines signals they completed the current wave and are now paused.
126 // Advance the current wave and mark the state as paused.
127 DpControlMessage::WaveComplete(wave) => {
rust/src/engine-core-client/src/error.rs
+2
@@ -25,6 +25,8 @@ pub enum Error {
25 ValueDecode(#[from] rmpv::decode::Error),
26 #[error("messagepack ext value decode failed: {message}")]
27 ExtValueDecode { message: String },
28 + #[error("invalid structured outputs params: {message}")]
29 + InvalidStructuredOutputsParams { message: String },
30 #[error("io error")]
31 Io(#[from] std::io::Error),
32 #[error("transport error")]
rust/src/engine-core-client/src/protocol/logprobs/tests.rs
+4 -4
@@ -183,7 +183,7 @@ fn decodes_inline_new_logprobs() {
183 Some(inline_logprobs_value()),
184 None,
185 )))];
186 - let decoded = decode_engine_core_outputs(&frames).unwrap();
186 + let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
187
188 let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
189 assert_eq!(logprobs, expected_sample_logprobs());
@@ -214,7 +214,7 @@ fn decodes_multipart_new_logprobs() {
214 ]),
215 Bytes::from_static(&[1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0]),
216 ];
217 - let decoded = decode_engine_core_outputs(&frames).unwrap();
217 + let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
218
219 let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
220 assert_eq!(logprobs, expected_sample_logprobs());
@@ -226,7 +226,7 @@ fn decodes_inline_prompt_logprobs() {
226 None,
227 Some(inline_prompt_logprobs_value()),
228 )))];
229 - let decoded = decode_engine_core_outputs(&frames).unwrap();
229 + let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
230
231 let logprobs = decoded.outputs[0]
232 .new_prompt_logprobs_tensors
@@ -252,7 +252,7 @@ fn decodes_big_endian_payloads() {
252 ])),
253 None,
254 )))];
255 - let decoded = decode_engine_core_outputs(&frames).unwrap();
255 + let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
256 let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
257 assert_eq!(
258 logprobs,
rust/src/engine-core-client/src/protocol/output.rs
+165 -115
@@ -1,7 +1,7 @@
1 use std::collections::BTreeSet;
2
3 use enum_as_inner::EnumAsInner;
4 -use serde::{Deserialize, Serialize};
4 +use serde::{Deserialize, Deserializer, Serialize, Serializer};
5 use serde_default::DefaultFromSerde;
6 use serde_repr::{Deserialize_repr, Serialize_repr};
7 use serde_tuple::{Deserialize_tuple, Serialize_tuple};
@@ -131,33 +131,33 @@ impl EngineCoreOutput {
131 }
132 }
133
134 -/// Batch of engine-core outputs returned to a frontend client.
134 +/// Raw Python/msgpack engine-core output envelope.
135 ///
136 /// Original Python definition:
137 /// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
138 #[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
139 -pub struct EngineCoreOutputs {
139 +struct WireEngineCoreOutputs {
140 #[serde(default)]
141 - pub engine_index: u32,
141 + engine_index: u32,
142 /// Outputs grouped for this client in the current engine tick.
143 #[serde(default)]
144 - pub outputs: Vec<EngineCoreOutput>,
144 + outputs: Vec<EngineCoreOutput>,
145 #[serde(default)]
146 - pub scheduler_stats: Option<Box<SchedulerStats>>,
146 + scheduler_stats: Option<Box<SchedulerStats>>,
147 #[serde(default)]
148 - pub timestamp: f64,
148 + timestamp: f64,
149 #[serde(default)]
150 - pub utility_output: Option<UtilityOutput>,
150 + utility_output: Option<UtilityOutput>,
151 #[serde(default)]
152 - pub finished_requests: Option<BTreeSet<String>>,
152 + finished_requests: Option<BTreeSet<String>>,
153 /// In DP mode, signals that the current wave finished and engines are
154 /// paused.
155 #[serde(default)]
156 - pub wave_complete: Option<u32>,
156 + wave_complete: Option<u32>,
157 /// In DP mode, signals that a request arrived for an old wave and the next
158 /// wave needs to start in other engines.
159 #[serde(default)]
160 - pub start_wave: Option<u32>,
160 + start_wave: Option<u32>,
161 }
162
163 /// Data-parallel control notifications multiplexed through `EngineCoreOutputs`.
@@ -167,7 +167,7 @@ pub enum DpControlMessage {
167 StartWave(u32),
168 }
169
170 -#[derive(Debug, Clone, PartialEq)]
170 +#[derive(Debug, Clone, PartialEq, Default)]
171 pub struct RequestBatchOutputs {
172 pub engine_index: u32,
173 pub outputs: Vec<EngineCoreOutput>,
@@ -176,30 +176,48 @@ pub struct RequestBatchOutputs {
176 pub finished_requests: Option<BTreeSet<String>>,
177 }
178
179 -#[derive(Debug, Clone, PartialEq)]
179 +#[derive(Debug, Clone, PartialEq, Default)]
180 pub struct UtilityCallOutput {
181 pub engine_index: u32,
182 pub timestamp: f64,
183 pub output: UtilityOutput,
184 }
185
186 -/// Semantic classification of a raw `EngineCoreOutputs` message.
186 +#[derive(Debug, Clone, PartialEq)]
187 +pub struct DpControlOutput {
188 + pub engine_index: u32,
189 + pub timestamp: f64,
190 + pub control: DpControlMessage,
191 +}
192 +
193 +/// Semantic engine-core output families.
194 ///
188 -/// Python currently uses one product-shaped wire struct for several distinct
189 -/// output families. This enum exposes those families more explicitly without
190 -/// changing the wire format.
195 +/// Python currently uses one product-shaped wire struct. The Rust protocol
196 +/// exposes the finite semantic families while preserving the same msgpack shape
197 +/// for serialization.
198 #[derive(Debug, Clone, PartialEq, EnumAsInner)]
192 -pub enum ClassifiedEngineCoreOutputs {
199 +pub enum EngineCoreOutputs {
200 RequestBatch(RequestBatchOutputs),
201 Utility(UtilityCallOutput),
195 - DpControl {
196 - engine_index: u32,
197 - timestamp: f64,
198 - control: DpControlMessage,
199 - },
200 - /// Fallback for wire-shape combinations that do not map cleanly onto the
201 - /// current semantic families.
202 - Other(EngineCoreOutputs),
202 + DpControl(DpControlOutput),
203 +}
204 +
205 +impl From<RequestBatchOutputs> for EngineCoreOutputs {
206 + fn from(outputs: RequestBatchOutputs) -> Self {
207 + Self::RequestBatch(outputs)
208 + }
209 +}
210 +
211 +impl From<UtilityCallOutput> for EngineCoreOutputs {
212 + fn from(output: UtilityCallOutput) -> Self {
213 + Self::Utility(output)
214 + }
215 +}
216 +
217 +impl From<DpControlOutput> for EngineCoreOutputs {
218 + fn from(output: DpControlOutput) -> Self {
219 + Self::DpControl(output)
220 + }
221 }
222
223 impl EngineCoreOutputs {
@@ -209,55 +227,119 @@ impl EngineCoreOutputs {
227 where
228 Frame: AsRef<[u8]>,
229 {
212 - for output in &mut self.outputs {
213 - output.resolve_in_place(frames)?;
230 + if let Self::RequestBatch(batch) = self {
231 + for output in &mut batch.outputs {
232 + output.resolve_in_place(frames)?;
233 + }
234 }
235 Ok(())
236 }
237 +}
238
218 - /// Classify the raw wire message into a more semantic Rust enum.
219 - pub fn classify(self) -> ClassifiedEngineCoreOutputs {
220 - let has_request_payload = !self.outputs.is_empty()
221 - || self.scheduler_stats.is_some()
222 - || self.finished_requests.is_some();
239 +/// Classify the raw wire message into a more semantic Rust enum.
240 +impl TryFrom<WireEngineCoreOutputs> for EngineCoreOutputs {
241 + type Error = Error;
242 +
243 + fn try_from(value: WireEngineCoreOutputs) -> Result<Self> {
244 + let has_request_payload = !value.outputs.is_empty()
245 + || value.scheduler_stats.is_some()
246 + || value.finished_requests.is_some();
247
248 match (
249 has_request_payload,
226 - &self.utility_output,
227 - &self.wave_complete,
228 - &self.start_wave,
250 + &value.utility_output,
251 + &value.wave_complete,
252 + &value.start_wave,
253 ) {
230 - (true, None, None, None) => {
231 - ClassifiedEngineCoreOutputs::RequestBatch(RequestBatchOutputs {
232 - engine_index: self.engine_index,
233 - outputs: self.outputs,
234 - scheduler_stats: self.scheduler_stats,
235 - timestamp: self.timestamp,
236 - finished_requests: self.finished_requests,
237 - })
254 + (true, None, None, None) => Ok(RequestBatchOutputs {
255 + engine_index: value.engine_index,
256 + outputs: value.outputs,
257 + scheduler_stats: value.scheduler_stats,
258 + timestamp: value.timestamp,
259 + finished_requests: value.finished_requests,
260 + }
261 + .into()),
262 + (false, Some(_), None, None) => Ok(UtilityCallOutput {
263 + engine_index: value.engine_index,
264 + timestamp: value.timestamp,
265 + output: value.utility_output.unwrap(),
266 }
239 - (false, Some(_), None, None) => {
240 - ClassifiedEngineCoreOutputs::Utility(UtilityCallOutput {
241 - engine_index: self.engine_index,
242 - timestamp: self.timestamp,
243 - output: self.utility_output.unwrap(),
244 - })
267 + .into()),
268 + (false, None, Some(_), None) => Ok(DpControlOutput {
269 + engine_index: value.engine_index,
270 + timestamp: value.timestamp,
271 + control: DpControlMessage::WaveComplete(value.wave_complete.unwrap()),
272 }
246 - (false, None, Some(_), None) => ClassifiedEngineCoreOutputs::DpControl {
247 - engine_index: self.engine_index,
248 - timestamp: self.timestamp,
249 - control: DpControlMessage::WaveComplete(self.wave_complete.unwrap()),
273 + .into()),
274 + (false, None, None, Some(_)) => Ok(DpControlOutput {
275 + engine_index: value.engine_index,
276 + timestamp: value.timestamp,
277 + control: DpControlMessage::StartWave(value.start_wave.unwrap()),
278 + }
279 + .into()),
280 +
281 + _ => Err(Error::Decode {
282 + target_type: "EngineCoreOutputs",
283 + message: "invalid wire shape".to_string(),
284 + }),
285 + }
286 + }
287 +}
288 +
289 +impl From<EngineCoreOutputs> for WireEngineCoreOutputs {
290 + fn from(value: EngineCoreOutputs) -> Self {
291 + match value {
292 + EngineCoreOutputs::RequestBatch(batch) => Self {
293 + engine_index: batch.engine_index,
294 + outputs: batch.outputs,
295 + scheduler_stats: batch.scheduler_stats,
296 + timestamp: batch.timestamp,
297 + finished_requests: batch.finished_requests,
298 + ..Default::default()
299 },
251 - (false, None, None, Some(_)) => ClassifiedEngineCoreOutputs::DpControl {
252 - engine_index: self.engine_index,
253 - timestamp: self.timestamp,
254 - control: DpControlMessage::StartWave(self.start_wave.unwrap()),
300 + EngineCoreOutputs::Utility(utility) => Self {
301 + engine_index: utility.engine_index,
302 + timestamp: utility.timestamp,
303 + utility_output: Some(utility.output),
304 + ..Default::default()
305 },
256 - _ => ClassifiedEngineCoreOutputs::Other(self),
306 + EngineCoreOutputs::DpControl(control) => {
307 + let (wave_complete, start_wave) = match control.control {
308 + DpControlMessage::WaveComplete(wave) => (Some(wave), None),
309 + DpControlMessage::StartWave(wave) => (None, Some(wave)),
310 + };
311 + Self {
312 + engine_index: control.engine_index,
313 + timestamp: control.timestamp,
314 + wave_complete,
315 + start_wave,
316 + ..Default::default()
317 + }
318 + }
319 }
320 }
321 }
322
323 +impl Serialize for EngineCoreOutputs {
324 + fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
325 + where
326 + S: Serializer,
327 + {
328 + WireEngineCoreOutputs::from(self.clone()).serialize(serializer)
329 + }
330 +}
331 +
332 +impl<'de> Deserialize<'de> for EngineCoreOutputs {
333 + fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
334 + where
335 + D: Deserializer<'de>,
336 + {
337 + WireEngineCoreOutputs::deserialize(deserializer)?
338 + .try_into()
339 + .map_err(serde::de::Error::custom)
340 + }
341 +}
342 +
343 /// Decode one ordinary or multipart engine-core output message into the strong
344 /// typed public protocol shape.
345 pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
@@ -281,7 +363,7 @@ mod tests {
363
364 #[test]
365 fn engine_core_outputs_roundtrip_finished_fields() {
284 - let outputs = EngineCoreOutputs {
366 + let outputs = WireEngineCoreOutputs {
367 outputs: vec![EngineCoreOutput {
368 request_id: "req-1".to_string(),
369 new_token_ids: vec![42],
@@ -302,7 +384,7 @@ mod tests {
384 };
385
386 let encoded = encode_msgpack(&outputs).unwrap();
305 - let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
387 + let decoded: WireEngineCoreOutputs = decode_msgpack(&encoded).unwrap();
388
389 assert_eq!(decoded.outputs.len(), 1);
390 assert_eq!(
@@ -317,7 +399,7 @@ mod tests {
399
400 #[test]
401 fn engine_core_outputs_classify_request_batch() {
320 - let outputs = EngineCoreOutputs {
402 + let outputs = WireEngineCoreOutputs {
403 outputs: vec![EngineCoreOutput {
404 request_id: "req-1".to_string(),
405 new_token_ids: vec![7],
@@ -360,12 +442,12 @@ mod tests {
442 },
443 )
444 "#]]
363 - .assert_debug_eq(&outputs.classify());
445 + .assert_debug_eq(&EngineCoreOutputs::try_from(outputs).unwrap());
446 }
447
448 #[test]
449 fn engine_core_outputs_classify_utility() {
368 - let outputs = EngineCoreOutputs {
450 + let outputs = WireEngineCoreOutputs {
451 utility_output: Some(UtilityOutput {
452 call_id: 42_u64.into(),
453 failure_message: None,
@@ -387,31 +469,33 @@ mod tests {
469 },
470 )
471 "#]]
390 - .assert_debug_eq(&outputs.classify());
472 + .assert_debug_eq(&EngineCoreOutputs::try_from(outputs).unwrap());
473 }
474
475 #[test]
476 fn engine_core_outputs_classify_control() {
395 - let outputs = EngineCoreOutputs {
477 + let outputs = WireEngineCoreOutputs {
478 start_wave: Some(3),
479 ..Default::default()
480 };
481
482 expect_test::expect![[r#"
401 - DpControl {
402 - engine_index: 0,
403 - timestamp: 0.0,
404 - control: StartWave(
405 - 3,
406 - ),
407 - }
483 + DpControl(
484 + DpControlOutput {
485 + engine_index: 0,
486 + timestamp: 0.0,
487 + control: StartWave(
488 + 3,
489 + ),
490 + },
491 + )
492 "#]]
409 - .assert_debug_eq(&outputs.classify());
493 + .assert_debug_eq(&EngineCoreOutputs::try_from(outputs).unwrap());
494 }
495
496 #[test]
413 - fn engine_core_outputs_classify_mixed_shape_as_raw() {
414 - let outputs = EngineCoreOutputs {
497 + fn engine_core_outputs_rejects_mixed_shape() {
498 + let outputs = WireEngineCoreOutputs {
499 outputs: vec![EngineCoreOutput {
500 request_id: "req-1".to_string(),
501 new_token_ids: vec![7],
@@ -425,44 +509,10 @@ mod tests {
509 ..Default::default()
510 };
511
428 - expect_test::expect![[r#"
429 - Other(
430 - EngineCoreOutputs {
431 - engine_index: 0,
432 - outputs: [
433 - EngineCoreOutput {
434 - request_id: "req-1",
435 - new_token_ids: [
436 - 7,
437 - ],
438 - new_logprobs: None,
439 - new_prompt_logprobs_tensors: None,
440 - pooling_output: None,
441 - finish_reason: None,
442 - stop_reason: None,
443 - events: None,
444 - kv_transfer_params: None,
445 - trace_headers: None,
446 - prefill_stats: None,
447 - routed_experts: None,
448 - num_nans_in_logits: 0,
449 - },
450 - ],
451 - scheduler_stats: None,
452 - timestamp: 0.0,
453 - utility_output: Some(
454 - UtilityOutput {
455 - call_id: 1,
456 - failure_message: None,
457 - result: None,
458 - },
459 - ),
460 - finished_requests: None,
461 - wave_complete: None,
462 - start_wave: None,
463 - },
464 - )
465 - "#]]
466 - .assert_debug_eq(&outputs.classify());
512 + let error = EngineCoreOutputs::try_from(outputs).unwrap_err();
513 + expect_test::expect![[
514 + r#"messagepack decode failed for EngineCoreOutputs: invalid wire shape"#
515 + ]]
516 + .assert_eq(&error.to_string());
517 }
518 }
rust/src/engine-core-client/src/protocol/structured_outputs.rs
+249 -25
@@ -1,4 +1,8 @@
1 -use serde::{Deserialize, Serialize};
1 +use enum_as_inner::EnumAsInner;
2 +use serde::{Deserialize, Deserializer, Serialize, Serializer};
3 +use serde_json::Value;
4 +
5 +use crate::error::{Error, Result};
6
7 /// Structured-output backend selected for EngineCore grammar compilation.
8 ///
@@ -15,50 +19,219 @@ pub enum StructuredOutputBackend {
19 LmFormatEnforcer,
20 }
21
18 -/// Parameters for configuring structured outputs (guided decoding).
19 -///
20 -/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
21 -/// `json_object`, or `structural_tag`) should be set. The engine-core
22 -/// backend selects the appropriate grammar compiler based on which field
23 -/// is present.
24 -///
25 -/// Original Python definition:
26 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
27 -#[serde_with::skip_serializing_none]
28 -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
29 -#[serde(default)]
30 -pub struct StructuredOutputsParams {
22 +/// The single structured-output constraint selected for a request.
23 +#[derive(Debug, Clone, PartialEq, EnumAsInner)]
24 +pub enum StructuredOutputConstraint {
25 /// JSON schema (as a dict/object or JSON string) constraining the output.
32 - pub json: Option<serde_json::Value>,
26 + Json(Value),
27 /// Regular expression the output must match.
34 - pub regex: Option<String>,
28 + Regex(String),
29 /// List of allowed output strings (the model must produce one of these).
36 - pub choice: Option<Vec<String>>,
30 + Choice(Vec<String>),
31 /// Context-free grammar (in EBNF-like notation) the output must conform to.
38 - pub grammar: Option<String>,
39 - /// When `true`, output must be valid JSON (free-form, no schema).
40 - pub json_object: Option<bool>,
32 + Grammar(String),
33 + /// Output must be valid JSON (free-form, no schema).
34 + JsonObject,
35 + /// Structural tag configuration (JSON-encoded string).
36 + StructuralTag(String),
37 +}
38 +
39 +/// Additional structured-output options that do not select the constraint mode.
40 +#[derive(Debug, Clone, Default, PartialEq, Eq)]
41 +pub struct StructuredOutputOptions {
42 /// Disable any additional whitespace in guided JSON output.
42 - #[serde(skip_serializing_if = "crate::protocol::is_false")]
43 pub disable_any_whitespace: bool,
44 /// Disable `additionalProperties` in JSON schema output.
45 - #[serde(skip_serializing_if = "crate::protocol::is_false")]
45 pub disable_additional_properties: bool,
46 /// Custom whitespace pattern for guided JSON output.
47 pub whitespace_pattern: Option<String>,
49 - /// Structural tag configuration (JSON-encoded string).
50 - pub structural_tag: Option<String>,
48 +}
49 +
50 +/// Parameters for configuring structured outputs (guided decoding).
51 +///
52 +/// This is the semantic Rust representation: exactly one constraint mode is
53 +/// always selected. The Python/msgpack product-shaped representation is kept in
54 +/// the private wire type below and used only at serde boundaries.
55 +///
56 +/// Original Python definition:
57 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
58 +#[derive(Debug, Clone, PartialEq)]
59 +pub struct StructuredOutputsParams {
60 + pub constraint: StructuredOutputConstraint,
61 + pub options: StructuredOutputOptions,
62 /// Structured-output backend, mirroring Python's internal `_backend`.
63 ///
64 /// User-supplied values are ignored during deserialization. This matches
65 /// Python's request boundary, where `_backend` is set by validation rather
66 /// than accepted as a request-level backend selector.
67 + pub backend: StructuredOutputBackend,
68 +}
69 +
70 +impl StructuredOutputsParams {
71 + pub fn json(json: Value) -> Self {
72 + Self::from_constraint(StructuredOutputConstraint::Json(json))
73 + }
74 +
75 + pub fn regex(regex: impl Into<String>) -> Self {
76 + Self::from_constraint(StructuredOutputConstraint::Regex(regex.into()))
77 + }
78 +
79 + pub fn choice(choice: Vec<String>) -> Self {
80 + Self::from_constraint(StructuredOutputConstraint::Choice(choice))
81 + }
82 +
83 + pub fn grammar(grammar: impl Into<String>) -> Self {
84 + Self::from_constraint(StructuredOutputConstraint::Grammar(grammar.into()))
85 + }
86 +
87 + pub fn json_object() -> Self {
88 + Self::from_constraint(StructuredOutputConstraint::JsonObject)
89 + }
90 +
91 + pub fn structural_tag(structural_tag: impl Into<String>) -> Self {
92 + Self::from_constraint(StructuredOutputConstraint::StructuralTag(
93 + structural_tag.into(),
94 + ))
95 + }
96 +
97 + fn from_constraint(constraint: StructuredOutputConstraint) -> Self {
98 + Self {
99 + constraint,
100 + options: StructuredOutputOptions::default(),
101 + backend: StructuredOutputBackend::default(),
102 + }
103 + }
104 +}
105 +
106 +/// Wire-compatible structured-output payload used by Python engine-core.
107 +///
108 +/// Python models `StructuredOutputsParams` as a product-shaped dataclass with
109 +/// several optional constraint fields, then validates that exactly one of those
110 +/// fields is present. Rust exposes [`StructuredOutputsParams`] as an enum-backed
111 +/// domain type instead, while using this private wire type for ser/de.
112 +#[serde_with::skip_serializing_none]
113 +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
114 +#[serde(default)]
115 +struct WireStructuredOutputsParams {
116 + json: Option<Value>,
117 + regex: Option<String>,
118 + choice: Option<Vec<String>>,
119 + grammar: Option<String>,
120 + json_object: Option<bool>,
121 + #[serde(skip_serializing_if = "crate::protocol::is_false")]
122 + disable_any_whitespace: bool,
123 + #[serde(skip_serializing_if = "crate::protocol::is_false")]
124 + disable_additional_properties: bool,
125 + whitespace_pattern: Option<String>,
126 + structural_tag: Option<String>,
127 #[serde(
128 default,
129 rename = "_backend",
130 deserialize_with = "serde_with::rust::deserialize_ignore_any"
131 )]
61 - pub backend: StructuredOutputBackend,
132 + backend: StructuredOutputBackend,
133 +}
134 +
135 +impl TryFrom<WireStructuredOutputsParams> for StructuredOutputsParams {
136 + type Error = Error;
137 +
138 + fn try_from(raw: WireStructuredOutputsParams) -> Result<Self> {
139 + use StructuredOutputConstraint::*;
140 +
141 + let mut constraint = None;
142 +
143 + macro_rules! insert_constraint {
144 + ($name:literal, $value:expr) => {
145 + if let Some(value) = $value {
146 + if let Some((existing, _)) = constraint {
147 + return Err(Error::InvalidStructuredOutputsParams {
148 + message: format!(
149 + "multiple structured output constraints specified: {existing}, {}",
150 + $name
151 + ),
152 + });
153 + }
154 + constraint = Some(($name, value));
155 + }
156 + };
157 + }
158 +
159 + insert_constraint!("json", raw.json.map(Json));
160 + insert_constraint!("regex", raw.regex.map(Regex));
161 + insert_constraint!("choice", raw.choice.map(Choice));
162 + insert_constraint!("grammar", raw.grammar.map(Grammar));
163 + match raw.json_object {
164 + Some(true) => {
165 + insert_constraint!("json_object", Some(JsonObject))
166 + }
167 + Some(false) => {
168 + return Err(Error::InvalidStructuredOutputsParams {
169 + message: "structured_outputs.json_object must be true if set; omit structured_outputs to disable structured outputs".to_string(),
170 + });
171 + }
172 + None => {}
173 + }
174 + insert_constraint!("structural_tag", raw.structural_tag.map(StructuralTag));
175 +
176 + Ok(Self {
177 + constraint: constraint.map(|(_, c)| c).ok_or_else(|| {
178 + Error::InvalidStructuredOutputsParams {
179 + message: "missing structured output constraint".to_string(),
180 + }
181 + })?,
182 + options: StructuredOutputOptions {
183 + disable_any_whitespace: raw.disable_any_whitespace,
184 + disable_additional_properties: raw.disable_additional_properties,
185 + whitespace_pattern: raw.whitespace_pattern,
186 + },
187 + backend: raw.backend,
188 + })
189 + }
190 +}
191 +
192 +impl From<StructuredOutputsParams> for WireStructuredOutputsParams {
193 + fn from(params: StructuredOutputsParams) -> Self {
194 + let mut raw = Self {
195 + disable_any_whitespace: params.options.disable_any_whitespace,
196 + disable_additional_properties: params.options.disable_additional_properties,
197 + whitespace_pattern: params.options.whitespace_pattern,
198 + backend: params.backend,
199 + ..Self::default()
200 + };
201 +
202 + match params.constraint {
203 + StructuredOutputConstraint::Json(json) => raw.json = Some(json),
204 + StructuredOutputConstraint::Regex(regex) => raw.regex = Some(regex),
205 + StructuredOutputConstraint::Choice(choice) => raw.choice = Some(choice),
206 + StructuredOutputConstraint::Grammar(grammar) => raw.grammar = Some(grammar),
207 + StructuredOutputConstraint::JsonObject => raw.json_object = Some(true),
208 + StructuredOutputConstraint::StructuralTag(structural_tag) => {
209 + raw.structural_tag = Some(structural_tag);
210 + }
211 + }
212 +
213 + raw
214 + }
215 +}
216 +
217 +impl Serialize for StructuredOutputsParams {
218 + fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
219 + where
220 + S: Serializer,
221 + {
222 + WireStructuredOutputsParams::from(self.clone()).serialize(serializer)
223 + }
224 +}
225 +
226 +impl<'de> Deserialize<'de> for StructuredOutputsParams {
227 + fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
228 + where
229 + D: Deserializer<'de>,
230 + {
231 + WireStructuredOutputsParams::deserialize(deserializer)?
232 + .try_into()
233 + .map_err(serde::de::Error::custom)
234 + }
235 }
236
237 #[cfg(test)]
@@ -74,8 +247,59 @@ mod tests {
247 .unwrap();
248
249 assert_eq!(params.backend, StructuredOutputBackend::Guidance);
250 + assert_eq!(params.constraint, StructuredOutputConstraint::JsonObject);
251
252 let value = serde_json::to_value(params).unwrap();
253 assert_eq!(value["_backend"], "guidance");
254 }
255 +
256 + #[test]
257 + fn structured_outputs_rejects_missing_constraint() {
258 + let error =
259 + serde_json::from_value::<StructuredOutputsParams>(serde_json::json!({})).unwrap_err();
260 +
261 + assert!(error.to_string().contains("missing structured output constraint"));
262 + }
263 +
264 + #[test]
265 + fn structured_outputs_rejects_multiple_constraints() {
266 + let error = serde_json::from_value::<StructuredOutputsParams>(serde_json::json!({
267 + "json": {"type": "object"},
268 + "regex": ".*",
269 + }))
270 + .unwrap_err();
271 +
272 + assert!(
273 + error
274 + .to_string()
275 + .contains("multiple structured output constraints specified: json, regex")
276 + );
277 + }
278 +
279 + #[test]
280 + fn structured_outputs_rejects_json_object_false() {
281 + let error = serde_json::from_value::<StructuredOutputsParams>(serde_json::json!({
282 + "json_object": false,
283 + }))
284 + .unwrap_err();
285 +
286 + assert!(error.to_string().contains("json_object must be true"));
287 + }
288 +
289 + #[test]
290 + fn structured_outputs_serializes_through_raw_shape() {
291 + let params = StructuredOutputsParams {
292 + constraint: StructuredOutputConstraint::StructuralTag(
293 + r#"{"type":"structural_tag"}"#.to_string(),
294 + ),
295 + options: StructuredOutputOptions::default(),
296 + backend: StructuredOutputBackend::Xgrammar,
297 + };
298 +
299 + let value = serde_json::to_value(params).unwrap();
300 +
301 + assert_eq!(value["structural_tag"], r#"{"type":"structural_tag"}"#);
302 + assert_eq!(value["_backend"], "xgrammar");
303 + assert!(value.get("json").is_none());
304 + }
305 }
rust/src/engine-core-client/src/tests/client.rs
+163 -219
@@ -23,7 +23,8 @@ use crate::protocol::multimodal::{
23 SliceSpec,
24 };
25 use crate::protocol::output::{
26 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, decode_engine_core_outputs,
26 + DpControlMessage, DpControlOutput, EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
27 + RequestBatchOutputs, UtilityCallOutput, decode_engine_core_outputs,
28 };
29 use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
30 use crate::protocol::sampling::EngineCoreSamplingParams;
@@ -571,8 +572,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
572
573 send_outputs(
574 &mut data_socket.push,
574 - EngineCoreOutputs {
575 - engine_index: 0,
575 + RequestBatchOutputs {
576 outputs: vec![request_output(
577 "req-1",
578 vec![],
@@ -580,17 +580,19 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
580 )],
581 finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
582 ..Default::default()
583 - },
583 + }
584 + .into(),
585 )
586 .await;
587
588 send_outputs(
589 &mut coordinator.output_push,
589 - EngineCoreOutputs {
590 + DpControlOutput {
591 engine_index: 0,
591 - wave_complete: Some(0),
592 - ..Default::default()
593 - },
592 + timestamp: 0.0,
593 + control: DpControlMessage::WaveComplete(0),
594 + }
595 + .into(),
596 )
597 .await;
598
@@ -605,8 +607,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
607
608 send_outputs(
609 &mut data_socket.push,
608 - EngineCoreOutputs {
609 - engine_index: 0,
610 + RequestBatchOutputs {
611 outputs: vec![request_output(
612 "req-3",
613 vec![],
@@ -614,7 +615,8 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
615 )],
616 finished_requests: Some(BTreeSet::from(["req-3".to_string()])),
617 ..Default::default()
617 - },
618 + }
619 + .into(),
620 )
621 .await;
622
@@ -651,7 +653,7 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
653
654 send_outputs(
655 &mut data_socket.push,
654 - EngineCoreOutputs {
656 + RequestBatchOutputs {
657 engine_index: 1,
658 outputs: vec![request_output(
659 "req-2",
@@ -660,7 +662,8 @@ async fn coordinator_wave_control_tracks_pause_running_and_rebroadcasts() {
662 )],
663 finished_requests: Some(BTreeSet::from(["req-2".to_string()])),
664 ..Default::default()
663 - },
665 + }
666 + .into(),
667 )
668 .await;
669
@@ -766,11 +769,12 @@ async fn coordinator_rebroadcasts_engine_start_wave_control() {
769
770 send_outputs(
771 &mut coordinator.output_push,
769 - EngineCoreOutputs {
772 + DpControlOutput {
773 engine_index: 1,
771 - start_wave: Some(4),
772 - ..Default::default()
773 - },
774 + timestamp: 0.0,
775 + control: DpControlMessage::StartWave(4),
776 + }
777 + .into(),
778 )
779 .await;
780
@@ -821,15 +825,16 @@ async fn coordinator_accepts_stats_only_outputs() {
825
826 send_outputs(
827 &mut coordinator.output_push,
824 - EngineCoreOutputs {
825 - engine_index: 0,
828 + RequestBatchOutputs {
829 + outputs: Vec::new(),
830 scheduler_stats: Some(Box::new(SchedulerStats {
831 num_running_reqs: 1,
832 current_wave: 0,
833 ..Default::default()
834 })),
835 ..Default::default()
832 - },
836 + }
837 + .into(),
838 )
839 .await;
840
@@ -840,8 +845,7 @@ async fn coordinator_accepts_stats_only_outputs() {
845
846 send_outputs(
847 &mut data_socket.push,
843 - EngineCoreOutputs {
844 - engine_index: 0,
848 + RequestBatchOutputs {
849 outputs: vec![request_output(
850 "req-stats",
851 vec![],
@@ -849,7 +853,8 @@ async fn coordinator_accepts_stats_only_outputs() {
853 )],
854 finished_requests: Some(BTreeSet::from(["req-stats".to_string()])),
855 ..Default::default()
852 - },
856 + }
857 + .into(),
858 )
859 .await;
860
@@ -910,30 +915,34 @@ async fn client_fail_closes_when_main_output_path_receives_dp_control() {
915
916 send_outputs(
917 push,
913 - EngineCoreOutputs {
914 - utility_output: Some(UtilityOutput {
918 + UtilityCallOutput {
919 + output: UtilityOutput {
920 call_id: 1_u64.into(),
921 failure_message: None,
922 result: None,
918 - }),
923 + },
924 ..Default::default()
920 - },
925 + }
926 + .into(),
927 )
928 .await;
929 send_outputs(
930 push,
925 - EngineCoreOutputs {
926 - start_wave: Some(3),
927 - ..Default::default()
928 - },
931 + DpControlOutput {
932 + engine_index: 0,
933 + timestamp: 0.0,
934 + control: DpControlMessage::StartWave(3),
935 + }
936 + .into(),
937 )
938 .await;
939 send_outputs(
940 push,
933 - EngineCoreOutputs {
941 + RequestBatchOutputs {
942 outputs: vec![request_output("req-1", vec![999], None)],
943 ..Default::default()
936 - },
944 + }
945 + .into(),
946 )
947 .await;
948
@@ -985,91 +994,6 @@ async fn client_fail_closes_when_main_output_path_receives_dp_control() {
994 client.shutdown().await.unwrap();
995 }
996
988 -#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
989 -async fn client_fail_closes_when_main_output_path_receives_mixed_shape_output() {
990 - init_tracing();
991 - let ipc = IpcNamespace::new().unwrap();
992 - let handshake_address = ipc.handshake_endpoint();
993 - let engine_id = b"engine-0".to_vec();
994 -
995 - let (shutdown_tx, engine_task) = spawn_mock_engine_task(
996 - handshake_address.clone(),
997 - engine_id.clone(),
998 - |dealer, push| {
999 - Box::pin(async move {
1000 - let add_1 = recv_engine_message(dealer).await;
1001 - assert_eq!(add_1[0].as_ref(), &[0x00]);
1002 - let request_1: EngineCoreRequest = rmp_serde::from_slice(&add_1[1]).unwrap();
1003 - assert_eq!(request_1.client_index, 7);
1004 - assert_eq!(request_1.request_id, "req-1");
1005 -
1006 - let add_2 = recv_engine_message(dealer).await;
1007 - assert_eq!(add_2[0].as_ref(), &[0x00]);
1008 - let request_2: EngineCoreRequest = rmp_serde::from_slice(&add_2[1]).unwrap();
1009 - assert_eq!(request_2.client_index, 7);
1010 - assert_eq!(request_2.request_id, "req-2");
1011 -
1012 - send_outputs(
1013 - push,
1014 - EngineCoreOutputs {
1015 - utility_output: Some(UtilityOutput {
1016 - call_id: 1_u64.into(),
1017 - failure_message: None,
1018 - result: None,
1019 - }),
1020 - outputs: vec![request_output("req-1", vec![999], None)],
1021 - ..Default::default()
1022 - },
1023 - )
1024 - .await;
1025 -
1026 - tokio::time::sleep(Duration::from_millis(50)).await;
1027 - })
1028 - },
1029 - );
1030 -
1031 - let client = connect_client_with_ipc(
1032 - handshake_test_config(
1033 - handshake_address,
1034 - 1,
1035 - "test-model",
1036 - Duration::from_secs(2),
1037 - 7,
1038 - None,
1039 - ),
1040 - &ipc,
1041 - )
1042 - .await;
1043 - assert_eq!(client.engine_identities()[0], b"engine-0");
1044 - assert!(client.ready_responses()[0].max_model_len > 0);
1045 -
1046 - let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
1047 - let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
1048 -
1049 - let error_2 = timeout(Duration::from_secs(1), stream_2.next())
1050 - .await
1051 - .unwrap()
1052 - .unwrap()
1053 - .unwrap_err();
1054 - assert!(is_unexpected_dispatcher_output(&error_2));
1055 -
1056 - let error_1 = timeout(Duration::from_secs(1), stream_1.next())
1057 - .await
1058 - .unwrap()
1059 - .unwrap()
1060 - .unwrap_err();
1061 - assert!(is_unexpected_dispatcher_output(&error_1));
1062 -
1063 - assert!(matches!(
1064 - client.health_error().as_deref(),
1065 - Some(error) if is_unexpected_dispatcher_output(error)
1066 - ));
1067 -
1068 - let _ = shutdown_tx.send(());
1069 - engine_task.await.unwrap();
1070 - client.shutdown().await.unwrap();
1071 -}
1072 -
997 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
998 async fn duplicate_request_ids_are_rejected_without_sending_a_second_add() {
999 init_tracing();
@@ -1091,7 +1015,7 @@ async fn duplicate_request_ids_are_rejected_without_sending_a_second_add() {
1015
1016 send_outputs(
1017 push,
1094 - EngineCoreOutputs {
1018 + RequestBatchOutputs {
1019 outputs: vec![request_output(
1020 "req-1",
1021 vec![],
@@ -1099,7 +1023,8 @@ async fn duplicate_request_ids_are_rejected_without_sending_a_second_add() {
1023 )],
1024 finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
1025 ..Default::default()
1102 - },
1026 + }
1027 + .into(),
1028 )
1029 .await;
1030 })
@@ -1158,10 +1083,12 @@ async fn finished_requests_without_final_output_is_treated_as_unexpected_close()
1083
1084 send_outputs(
1085 push,
1161 - EngineCoreOutputs {
1086 + RequestBatchOutputs {
1087 + outputs: Vec::new(),
1088 finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
1089 ..Default::default()
1164 - },
1090 + }
1091 + .into(),
1092 )
1093 .await;
1094
@@ -1217,10 +1144,11 @@ async fn dropping_a_live_stream_triggers_abort() {
1144 assert_eq!(add[0].as_ref(), &[0x00]);
1145 send_outputs(
1146 push,
1220 - EngineCoreOutputs {
1147 + RequestBatchOutputs {
1148 outputs: vec![request_output("req-1", vec![99], None)],
1149 ..Default::default()
1223 - },
1150 + }
1151 + .into(),
1152 )
1153 .await;
1154
@@ -1275,14 +1203,15 @@ async fn dropping_multiple_live_streams_aborts_all_in_a_burst() {
1203 }
1204 send_outputs(
1205 push,
1278 - EngineCoreOutputs {
1206 + RequestBatchOutputs {
1207 outputs: vec![
1208 request_output("req-1", vec![99], None),
1209 request_output("req-2", vec![99], None),
1210 request_output("req-3", vec![99], None),
1211 ],
1212 ..Default::default()
1285 - },
1213 + }
1214 + .into(),
1215 )
1216 .await;
1217
@@ -1436,14 +1365,16 @@ async fn is_sleeping_wrapper_sends_typed_request_and_returns_typed_response() {
1365
1366 send_outputs(
1367 push,
1439 - EngineCoreOutputs {
1440 - utility_output: Some(UtilityOutput {
1368 + UtilityCallOutput {
1369 + engine_index: 0,
1370 + timestamp: 0.0,
1371 + output: UtilityOutput {
1372 call_id: call_id.into(),
1373 failure_message: None,
1374 result: Some(utility_result_value(true)),
1444 - }),
1445 - ..Default::default()
1446 - },
1375 + },
1376 + }
1377 + .into(),
1378 )
1379 .await;
1380 })
@@ -1491,14 +1422,16 @@ async fn call_utility_failure_message_surfaces_as_error() {
1422
1423 send_outputs(
1424 push,
1494 - EngineCoreOutputs {
1495 - utility_output: Some(UtilityOutput {
1425 + UtilityCallOutput {
1426 + engine_index: 0,
1427 + timestamp: 0.0,
1428 + output: UtilityOutput {
1429 call_id: call_id.into(),
1430 failure_message: Some("boom".to_string()),
1431 result: None,
1499 - }),
1500 - ..Default::default()
1501 - },
1432 + },
1433 + }
1434 + .into(),
1435 )
1436 .await;
1437 })
@@ -1815,8 +1748,7 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
1748 finish_req_1_rx.await.unwrap();
1749 send_outputs(
1750 push,
1818 - EngineCoreOutputs {
1819 - engine_index: 0,
1751 + RequestBatchOutputs {
1752 outputs: vec![request_output(
1753 &request_1.request_id,
1754 vec![10],
@@ -1824,7 +1756,8 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
1756 )],
1757 finished_requests: Some(BTreeSet::from([request_1.request_id.clone()])),
1758 ..Default::default()
1827 - },
1759 + }
1760 + .into(),
1761 )
1762 .await;
1763
@@ -1836,8 +1769,7 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
1769 finish_req_3_rx.await.unwrap();
1770 send_outputs(
1771 push,
1839 - EngineCoreOutputs {
1840 - engine_index: 0,
1772 + RequestBatchOutputs {
1773 outputs: vec![request_output(
1774 &request_3.request_id,
1775 vec![30],
@@ -1845,7 +1777,8 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
1777 )],
1778 finished_requests: Some(BTreeSet::from([request_3.request_id.clone()])),
1779 ..Default::default()
1848 - },
1780 + }
1781 + .into(),
1782 )
1783 .await;
1784 })
@@ -1864,7 +1797,7 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
1797 finish_req_2_rx.await.unwrap();
1798 send_outputs(
1799 push,
1867 - EngineCoreOutputs {
1800 + RequestBatchOutputs {
1801 engine_index: 1,
1802 outputs: vec![request_output(
1803 &request_2.request_id,
@@ -1873,7 +1806,8 @@ async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
1806 )],
1807 finished_requests: Some(BTreeSet::from([request_2.request_id.clone()])),
1808 ..Default::default()
1876 - },
1809 + }
1810 + .into(),
1811 )
1812 .await;
1813 })
@@ -1990,14 +1924,16 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
1924 assert_eq!(array[2], Value::from("is_sleeping"));
1925 send_outputs(
1926 push,
1993 - EngineCoreOutputs {
1994 - utility_output: Some(UtilityOutput {
1927 + UtilityCallOutput {
1928 + engine_index: 0,
1929 + timestamp: 0.0,
1930 + output: UtilityOutput {
1931 call_id: call_id.into(),
1932 failure_message: None,
1933 result: Some(utility_result_value(true)),
1998 - }),
1999 - ..Default::default()
2000 - },
1934 + },
1935 + }
1936 + .into(),
1937 )
1938 .await;
1939
@@ -2012,8 +1948,7 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
1948 assert_eq!(aborted_ids, vec!["req-1".to_string()]);
1949 send_outputs(
1950 push,
2015 - EngineCoreOutputs {
2016 - engine_index: 0,
1951 + RequestBatchOutputs {
1952 outputs: vec![request_output(
1953 "req-1",
1954 vec![],
@@ -2021,7 +1956,8 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
1956 )],
1957 finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
1958 ..Default::default()
2024 - },
1959 + }
1960 + .into(),
1961 )
1962 .await;
1963 })
@@ -2044,14 +1980,16 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
1980 assert_eq!(array[2], Value::from("is_sleeping"));
1981 send_outputs(
1982 push,
2047 - EngineCoreOutputs {
2048 - utility_output: Some(UtilityOutput {
1983 + UtilityCallOutput {
1984 + engine_index: 0,
1985 + timestamp: 0.0,
1986 + output: UtilityOutput {
1987 call_id: call_id.into(),
1988 failure_message: None,
1989 result: Some(utility_result_value(true)),
2052 - }),
2053 - ..Default::default()
2054 - },
1990 + },
1991 + }
1992 + .into(),
1993 )
1994 .await;
1995
@@ -2066,7 +2004,7 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
2004 assert_eq!(aborted_ids, vec!["req-2".to_string()]);
2005 send_outputs(
2006 push,
2069 - EngineCoreOutputs {
2007 + RequestBatchOutputs {
2008 engine_index: 1,
2009 outputs: vec![request_output(
2010 "req-2",
@@ -2075,7 +2013,8 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
2013 )],
2014 finished_requests: Some(BTreeSet::from(["req-2".to_string()])),
2015 ..Default::default()
2078 - },
2016 + }
2017 + .into(),
2018 )
2019 .await;
2020 })
@@ -2155,14 +2094,16 @@ async fn collective_rpc_flattens_results_from_all_engines() {
2094
2095 send_outputs(
2096 push,
2158 - EngineCoreOutputs {
2159 - utility_output: Some(UtilityOutput {
2097 + UtilityCallOutput {
2098 + engine_index: 0,
2099 + timestamp: 0.0,
2100 + output: UtilityOutput {
2101 call_id: call_id.into(),
2102 failure_message: None,
2103 result: Some(utility_result_value(vec!["engine-0-worker"])),
2163 - }),
2164 - ..Default::default()
2165 - },
2104 + },
2105 + }
2106 + .into(),
2107 )
2108 .await;
2109 })
@@ -2186,14 +2127,16 @@ async fn collective_rpc_flattens_results_from_all_engines() {
2127
2128 send_outputs(
2129 push,
2189 - EngineCoreOutputs {
2190 - utility_output: Some(UtilityOutput {
2130 + UtilityCallOutput {
2131 + engine_index: 0,
2132 + timestamp: 0.0,
2133 + output: UtilityOutput {
2134 call_id: call_id.into(),
2135 failure_message: None,
2136 result: Some(utility_result_value(vec!["engine-1-worker"])),
2194 - }),
2195 - ..Default::default()
2196 - },
2137 + },
2138 + }
2139 + .into(),
2140 )
2141 .await;
2142 })
@@ -2264,14 +2207,16 @@ fn spawn_mock_utility_engine(
2207 assert_eq!(array[3], expected_args, "unexpected utility args");
2208 send_outputs(
2209 push,
2267 - EngineCoreOutputs {
2268 - utility_output: Some(UtilityOutput {
2210 + UtilityCallOutput {
2211 + engine_index: 0,
2212 + timestamp: 0.0,
2213 + output: UtilityOutput {
2214 call_id: call_id.into(),
2215 failure_message: None,
2216 result: Some(utility_result_value(result)),
2272 - }),
2273 - ..Default::default()
2274 - },
2217 + },
2218 + }
2219 + .into(),
2220 )
2221 .await;
2222 })
@@ -2552,41 +2497,40 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2497
2498 let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap();
2499 expect_test::expect![[r#"
2555 - EngineCoreOutputs {
2556 - engine_index: 0,
2557 - outputs: [
2558 - EngineCoreOutput {
2559 - request_id: "req-1",
2560 - new_token_ids: [
2561 - 7,
2562 - 8,
2563 - ],
2564 - new_logprobs: None,
2565 - new_prompt_logprobs_tensors: None,
2566 - pooling_output: None,
2567 - finish_reason: Some(
2568 - Length,
2569 - ),
2570 - stop_reason: None,
2571 - events: None,
2572 - kv_transfer_params: None,
2573 - trace_headers: None,
2574 - prefill_stats: None,
2575 - routed_experts: None,
2576 - num_nans_in_logits: 0,
2577 - },
2578 - ],
2579 - scheduler_stats: None,
2580 - timestamp: 0.0,
2581 - utility_output: None,
2582 - finished_requests: Some(
2583 - {
2584 - "req-1",
2585 - },
2586 - ),
2587 - wave_complete: None,
2588 - start_wave: None,
2589 - }
2500 + RequestBatch(
2501 + RequestBatchOutputs {
2502 + engine_index: 0,
2503 + outputs: [
2504 + EngineCoreOutput {
2505 + request_id: "req-1",
2506 + new_token_ids: [
2507 + 7,
2508 + 8,
2509 + ],
2510 + new_logprobs: None,
2511 + new_prompt_logprobs_tensors: None,
2512 + pooling_output: None,
2513 + finish_reason: Some(
2514 + Length,
2515 + ),
2516 + stop_reason: None,
2517 + events: None,
2518 + kv_transfer_params: None,
2519 + trace_headers: None,
2520 + prefill_stats: None,
2521 + routed_experts: None,
2522 + num_nans_in_logits: 0,
2523 + },
2524 + ],
2525 + scheduler_stats: None,
2526 + timestamp: 0.0,
2527 + finished_requests: Some(
2528 + {
2529 + "req-1",
2530 + },
2531 + ),
2532 + },
2533 + )
2534 "#]]
2535 .assert_debug_eq(&decoded_outputs);
2536
@@ -2599,7 +2543,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2543 let inline_logprobs =
2544 decode_engine_core_outputs(&decode_frames(inline_logprobs_frames)).unwrap();
2545 expect_sample_logprobs(
2602 - inline_logprobs.outputs[0]
2546 + inline_logprobs.as_request_batch().unwrap().outputs[0]
2547 .new_logprobs
2548 .as_ref()
2549 .expect("inline logprobs decoded"),
@@ -2608,7 +2552,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2552 let multipart_logprobs =
2553 decode_engine_core_outputs(&decode_frames(multipart_logprobs_frames)).unwrap();
2554 expect_sample_logprobs(
2611 - multipart_logprobs.outputs[0]
2555 + multipart_logprobs.as_request_batch().unwrap().outputs[0]
2556 .new_logprobs
2557 .as_ref()
2558 .expect("multipart logprobs decoded"),
@@ -2616,7 +2560,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2560
2561 let inline_prompt = decode_engine_core_outputs(&decode_frames(inline_prompt_frames)).unwrap();
2562 expect_prompt_logprobs(
2619 - inline_prompt.outputs[0]
2563 + inline_prompt.as_request_batch().unwrap().outputs[0]
2564 .new_prompt_logprobs_tensors
2565 .as_ref()
2566 .expect("inline prompt logprobs decoded"),
@@ -2625,7 +2569,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
2569 let multipart_prompt =
2570 decode_engine_core_outputs(&decode_frames(multipart_prompt_frames)).unwrap();
2571 expect_prompt_logprobs(
2628 - multipart_prompt.outputs[0]
2572 + multipart_prompt.as_request_batch().unwrap().outputs[0]
2573 .new_prompt_logprobs_tensors
2574 .as_ref()
2575 .expect("multipart prompt logprobs decoded"),
@@ -2945,8 +2889,7 @@ async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends
2889
2890 send_outputs(
2891 &mut push,
2948 - EngineCoreOutputs {
2949 - engine_index: 0,
2892 + RequestBatchOutputs {
2893 outputs: vec![request_output(
2894 "req-1",
2895 vec![],
@@ -2954,7 +2897,8 @@ async fn bootstrapped_external_coordinator_updates_wave_ignores_counts_and_sends
2897 )],
2898 finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
2899 ..Default::default()
2957 - },
2900 + }
2901 + .into(),
2902 )
2903 .await;
2904
@@ -3020,8 +2964,7 @@ async fn bootstrapped_external_coordinator_running_state_suppresses_wakeup() {
2964
2965 send_outputs(
2966 &mut push,
3023 - EngineCoreOutputs {
3024 - engine_index: 0,
2967 + RequestBatchOutputs {
2968 outputs: vec![request_output(
2969 "req-1",
2970 vec![],
@@ -3029,7 +2972,8 @@ async fn bootstrapped_external_coordinator_running_state_suppresses_wakeup() {
2972 )],
2973 finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
2974 ..Default::default()
3032 - },
2975 + }
2976 + .into(),
2977 )
2978 .await;
2979
rust/src/llm/tests/generate.rs
+36 -30
@@ -11,7 +11,7 @@ use vllm_engine_core_client::protocol::logprobs::{
11 };
12 use vllm_engine_core_client::protocol::output::{
13 EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
14 - EngineCoreOutputs,
14 + EngineCoreOutputs, RequestBatchOutputs,
15 };
16 use vllm_engine_core_client::protocol::request::EngineCoreRequest;
17 use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
@@ -251,7 +251,7 @@ async fn generate_streams_outputs() {
251
252 send_outputs(
253 push,
254 - EngineCoreOutputs {
254 + RequestBatchOutputs {
255 outputs: vec![
256 request_output_with_logprobs(
257 &request.request_id,
@@ -270,7 +270,8 @@ async fn generate_streams_outputs() {
270 ],
271 finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
272 ..Default::default()
273 - },
273 + }
274 + .into(),
275 )
276 .await;
277 })
@@ -331,8 +332,7 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {
332
333 send_outputs(
334 push,
334 - EngineCoreOutputs {
335 - engine_index: 0,
335 + RequestBatchOutputs {
336 outputs: vec![
337 EngineCoreOutput {
338 prefill_stats: Some(PrefillStats {
@@ -358,13 +358,9 @@ async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {
358 Some(serde_json::json!({"connector": "x"})),
359 ),
360 ],
361 - scheduler_stats: None,
362 - timestamp: 0.0,
363 - utility_output: None,
364 - finished_requests: None,
365 - wave_complete: None,
366 - start_wave: None,
367 - },
361 + ..Default::default()
362 + }
363 + .into(),
364 )
365 .await;
366 })
@@ -412,10 +408,12 @@ async fn generate_propagates_unexpected_close_errors() {
408
409 send_outputs(
410 push,
415 - EngineCoreOutputs {
411 + RequestBatchOutputs {
412 + outputs: Vec::new(),
413 finished_requests: Some(BTreeSet::from([request.request_id])),
414 ..Default::default()
418 - },
415 + }
416 + .into(),
417 )
418 .await;
419 })
@@ -459,10 +457,11 @@ async fn dropping_a_live_generate_stream_triggers_abort() {
457
458 send_outputs(
459 push,
462 - EngineCoreOutputs {
460 + RequestBatchOutputs {
461 outputs: vec![request_output(&request.request_id, vec![99], None)],
462 ..Default::default()
465 - },
463 + }
464 + .into(),
465 )
466 .await;
467
@@ -513,7 +512,7 @@ async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_co
512
513 send_outputs(
514 push,
516 - EngineCoreOutputs {
515 + RequestBatchOutputs {
516 outputs: vec![request_output(
517 &request_1.request_id,
518 vec![],
@@ -521,13 +520,14 @@ async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_co
520 )],
521 finished_requests: Some(BTreeSet::from([request_1.request_id.clone()])),
522 ..Default::default()
524 - },
523 + }
524 + .into(),
525 )
526 .await;
527
528 send_outputs(
529 push,
530 - EngineCoreOutputs {
530 + RequestBatchOutputs {
531 outputs: vec![request_output(
532 &request_2.request_id,
533 vec![],
@@ -535,7 +535,8 @@ async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_co
535 )],
536 finished_requests: Some(BTreeSet::from([request_2.request_id])),
537 ..Default::default()
538 - },
538 + }
539 + .into(),
540 )
541 .await;
542 })
@@ -576,10 +577,11 @@ async fn abort_resolves_external_request_id_to_internal_before_reaching_engine()
577
578 send_outputs(
579 push,
579 - EngineCoreOutputs {
580 + RequestBatchOutputs {
581 outputs: vec![request_output(&request.request_id, vec![7], None)],
582 ..Default::default()
582 - },
583 + }
584 + .into(),
585 )
586 .await;
587
@@ -643,13 +645,14 @@ async fn abort_by_external_id_aborts_all_internal_requests() {
645
646 send_outputs(
647 push,
646 - EngineCoreOutputs {
648 + RequestBatchOutputs {
649 outputs: vec![
650 request_output(&request_1.request_id, vec![7], None),
651 request_output(&request_2.request_id, vec![8], None),
652 ],
653 ..Default::default()
652 - },
654 + }
655 + .into(),
656 )
657 .await;
658
@@ -710,7 +713,7 @@ async fn generate_records_request_metrics_in_prometheus_output() {
713
714 send_outputs(
715 push,
713 - EngineCoreOutputs {
716 + RequestBatchOutputs {
717 engine_index: 4,
718 timestamp: 10.0,
719 outputs: vec![EngineCoreOutput {
@@ -736,13 +739,14 @@ async fn generate_records_request_metrics_in_prometheus_output() {
739 )
740 }],
741 ..Default::default()
739 - },
742 + }
743 + .into(),
744 )
745 .await;
746
747 send_outputs(
748 push,
745 - EngineCoreOutputs {
749 + RequestBatchOutputs {
750 engine_index: 4,
751 timestamp: 11.5,
752 outputs: vec![request_output_with_events(
@@ -756,7 +760,8 @@ async fn generate_records_request_metrics_in_prometheus_output() {
760 )],
761 finished_requests: Some(BTreeSet::from([request.request_id])),
762 ..Default::default()
759 - },
763 + }
764 + .into(),
765 )
766 .await;
767 })
@@ -843,7 +848,7 @@ async fn dropping_stream_records_abort_terminal_request_metrics() {
848
849 send_outputs(
850 push,
846 - EngineCoreOutputs {
851 + RequestBatchOutputs {
852 engine_index: 5,
853 timestamp: 10.0,
854 outputs: vec![request_output_with_events(
@@ -862,7 +867,8 @@ async fn dropping_stream_records_abort_terminal_request_metrics() {
867 ]),
868 )],
869 ..Default::default()
865 - },
870 + }
871 + .into(),
872 )
873 .await;
874
rust/src/mock-engine/src/engine.rs
+16 -12
@@ -12,7 +12,8 @@ use tokio::task::yield_now;
12 use tokio_util::sync::CancellationToken;
13 use tracing::{debug, info, warn};
14 use vllm_engine_core_client::protocol::output::{
15 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
15 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
16 + UtilityCallOutput,
17 };
18 use vllm_engine_core_client::protocol::request::EngineCoreRequest;
19 use vllm_engine_core_client::protocol::utility::{
@@ -61,13 +62,14 @@ fn empty_finish_outputs(
62 let output = request_output(request_id, Vec::new(), Some(finish_reason));
63 let finished_requests = BTreeSet::from([output.request_id.clone()]);
64
64 - EngineCoreOutputs {
65 + RequestBatchOutputs {
66 engine_index,
67 outputs: vec![output],
68 timestamp: now_secs(),
69 finished_requests: Some(finished_requests),
70 ..Default::default()
71 }
72 + .into()
73 }
74
75 /// Encode a utility result into the protocol's msgpack value envelope.
@@ -98,16 +100,16 @@ fn utility_response(
100 _ => utility_envelope(Value::Nil),
101 }?;
102
101 - Ok(EngineCoreOutputs {
103 + Ok(UtilityCallOutput {
104 engine_index,
103 - utility_output: Some(UtilityOutput {
105 + timestamp: now_secs(),
106 + output: UtilityOutput {
107 call_id: request.call_id,
108 failure_message: None,
109 result: Some(result),
107 - }),
108 - timestamp: now_secs(),
109 - ..Default::default()
110 - })
110 + },
111 + }
112 + .into())
113 }
114
115 /// Message sent from the frontend to the mock engine task to drive the engine loop.
@@ -270,13 +272,14 @@ impl Engine {
272 }
273 for (client_index, (client_outputs, finished_requests)) in outputs_by_client {
274 outputs.push({
273 - let outputs = EngineCoreOutputs {
275 + let outputs = RequestBatchOutputs {
276 engine_index: self.engine_index,
277 outputs: client_outputs,
278 timestamp: now_secs(),
279 finished_requests: Some(finished_requests),
280 ..Default::default()
279 - };
281 + }
282 + .into();
283 EngineOutput {
284 client_index,
285 outputs,
@@ -358,14 +361,15 @@ impl Engine {
361 .filter_map(|(client_index, (outputs, finished_requests))| {
362 (!outputs.is_empty()).then(|| EngineOutput {
363 client_index,
361 - outputs: EngineCoreOutputs {
364 + outputs: RequestBatchOutputs {
365 engine_index: self.engine_index,
366 outputs,
367 timestamp: now_secs(),
368 finished_requests: (!finished_requests.is_empty())
369 .then_some(finished_requests),
370 ..Default::default()
368 - },
371 + }
372 + .into(),
373 })
374 })
375 .collect()
rust/src/server/src/grpc/convert.rs
+10 -24
@@ -231,32 +231,18 @@ fn convert_structured_output(
231 StructuredOutput::Json(schema) => {
232 let json: serde_json::Value = serde_json::from_str(schema)
233 .map_err(|e| Status::invalid_argument(format!("invalid json schema: {e}")))?;
234 - StructuredOutputsParams {
235 - json: Some(json),
236 - ..Default::default()
237 - }
234 + StructuredOutputsParams::json(json)
235 }
239 - StructuredOutput::Regex(regex) => StructuredOutputsParams {
240 - regex: Some(regex.clone()),
241 - ..Default::default()
242 - },
243 - StructuredOutput::Choice(choices) => StructuredOutputsParams {
244 - choice: Some(choices.choices.clone()),
245 - ..Default::default()
246 - },
247 - StructuredOutput::Grammar(grammar) => StructuredOutputsParams {
248 - grammar: Some(grammar.clone()),
249 - ..Default::default()
250 - },
251 - StructuredOutput::JsonObject(true) => StructuredOutputsParams {
252 - json_object: Some(true),
253 - ..Default::default()
254 - },
236 + StructuredOutput::Regex(regex) => StructuredOutputsParams::regex(regex.clone()),
237 + StructuredOutput::Choice(choices) => {
238 + StructuredOutputsParams::choice(choices.choices.clone())
239 + }
240 + StructuredOutput::Grammar(grammar) => StructuredOutputsParams::grammar(grammar.clone()),
241 + StructuredOutput::JsonObject(true) => StructuredOutputsParams::json_object(),
242 StructuredOutput::JsonObject(false) => return Ok(None),
256 - StructuredOutput::StructuralTag(tag) => StructuredOutputsParams {
257 - structural_tag: Some(tag.clone()),
258 - ..Default::default()
259 - },
243 + StructuredOutput::StructuralTag(tag) => {
244 + StructuredOutputsParams::structural_tag(tag.clone())
245 + }
246 };
247 Ok(Some(params))
248 }
rust/src/server/src/grpc/tests.rs
+4 -9
@@ -19,7 +19,7 @@ use vllm_chat::{
19 DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
20 };
21 use vllm_engine_core_client::protocol::output::{
22 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
22 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
23 };
24 use vllm_engine_core_client::protocol::request::EngineCoreRequest;
25 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
@@ -115,19 +115,14 @@ fn engine_outputs_for_request(
115 request_id: &str,
116 output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
117 ) -> EngineCoreOutputs {
118 - EngineCoreOutputs {
119 - engine_index: 0,
118 + RequestBatchOutputs {
119 outputs: output_specs
120 .into_iter()
121 .map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
122 .collect(),
124 - scheduler_stats: None,
125 - timestamp: 0.0,
126 - utility_output: None,
127 - finished_requests: None,
128 - wave_complete: None,
129 - start_wave: None,
123 + ..Default::default()
124 }
125 + .into()
126 }
127
128 fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
rust/src/server/src/routes/http_client_tests.rs
+4 -9
@@ -19,7 +19,7 @@ use vllm_chat::{
19 DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
20 };
21 use vllm_engine_core_client::protocol::output::{
22 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
22 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
23 };
24 use vllm_engine_core_client::protocol::request::EngineCoreRequest;
25 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
@@ -111,19 +111,14 @@ fn engine_outputs_for_request(
111 request_id: &str,
112 output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
113 ) -> EngineCoreOutputs {
114 - EngineCoreOutputs {
115 - engine_index: 0,
114 + RequestBatchOutputs {
115 outputs: output_specs
116 .into_iter()
117 .map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
118 .collect(),
120 - scheduler_stats: None,
121 - timestamp: 0.0,
122 - utility_output: None,
123 - finished_requests: None,
124 - wave_complete: None,
125 - start_wave: None,
119 + ..Default::default()
120 }
121 + .into()
122 }
123
124 fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
rust/src/server/src/routes/openai/utils/structured_outputs.rs
+6 -13
@@ -37,7 +37,7 @@ pub enum ResponseFormat {
37 },
38 /// vLLM-specific structural tag format. The entire object (including the
39 /// `type` field) is JSON-serialized and passed as
40 - /// `StructuredOutputsParams.structural_tag`.
40 + /// `StructuredOutputConstraint::StructuralTag`.
41 ///
42 /// We capture the payload as a catch-all map so both the legacy
43 /// (`structures`/`triggers`) and current (`format`) shapes are
@@ -79,14 +79,10 @@ pub fn convert_from_response_format(
79 };
80 match fmt {
81 ResponseFormat::Text => Ok(None),
82 - ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams {
83 - json_object: Some(true),
84 - ..Default::default()
85 - })),
86 - ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams {
87 - json: Some(json_schema.schema.clone()),
88 - ..Default::default()
89 - })),
82 + ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams::json_object())),
83 + ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams::json(
84 + json_schema.schema.clone(),
85 + ))),
86 ResponseFormat::StructuralTag { .. } => {
87 // The Python frontend dumps the entire response_format object (including the
88 // `type` field) as a JSON string for the engine-core backend.
@@ -96,10 +92,7 @@ pub fn convert_from_response_format(
92 Some("response_format"),
93 )
94 })?;
99 - Ok(Some(StructuredOutputsParams {
100 - structural_tag: Some(tag_json),
101 - ..Default::default()
102 - }))
95 + Ok(Some(StructuredOutputsParams::structural_tag(tag_json)))
96 }
97 }
98 }
rust/src/server/src/routes/tests.rs
+45 -91
@@ -29,7 +29,8 @@ use vllm_engine_core_client::protocol::logprobs::{
29 Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
30 };
31 use vllm_engine_core_client::protocol::output::{
32 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, StopReason,
32 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs, StopReason,
33 + UtilityCallOutput,
34 };
35 use vllm_engine_core_client::protocol::request::EngineCoreRequest;
36 use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
@@ -232,19 +233,14 @@ fn engine_outputs_for_request(
233 request_id: &str,
234 output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
235 ) -> EngineCoreOutputs {
235 - EngineCoreOutputs {
236 - engine_index: 0,
236 + RequestBatchOutputs {
237 outputs: output_specs
238 .into_iter()
239 .map(|(token_ids, finish_reason)| request_output(request_id, token_ids, finish_reason))
240 .collect(),
241 - scheduler_stats: None,
242 - timestamp: 0.0,
243 - utility_output: None,
244 - finished_requests: None,
245 - wave_complete: None,
246 - start_wave: None,
241 + ..Default::default()
242 }
243 + .into()
244 }
245
246 fn test_llm(client: EngineCoreClient) -> Llm {
@@ -390,14 +386,15 @@ fn utility_none_result() -> UtilityResultEnvelope {
386 }
387
388 fn utility_outputs(call_id: u64, result: UtilityResultEnvelope) -> EngineCoreOutputs {
393 - EngineCoreOutputs {
394 - utility_output: Some(UtilityOutput {
389 + UtilityCallOutput {
390 + output: UtilityOutput {
391 call_id: call_id.into(),
392 failure_message: None,
393 result: Some(result),
398 - }),
394 + },
395 ..Default::default()
396 }
397 + .into()
398 }
399
400 async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
@@ -2281,8 +2278,7 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
2278
2279 send_outputs(
2280 push,
2284 - EngineCoreOutputs {
2285 - engine_index: 0,
2281 + RequestBatchOutputs {
2282 outputs: vec![request_output_with_logprobs(
2283 &request.request_id,
2284 bytes_to_token_ids(b"hi"),
@@ -2291,13 +2287,9 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
2287 Some(sample_logprobs_for_tokens(&bytes_to_token_ids(b"hi"))),
2288 Some(prompt_logprobs_for_tokens(&prompt_token_ids)),
2289 )],
2294 - scheduler_stats: None,
2295 - timestamp: 0.0,
2296 - utility_output: None,
2297 - finished_requests: None,
2298 - wave_complete: None,
2299 - start_wave: None,
2300 - },
2290 + ..Default::default()
2291 + }
2292 + .into(),
2293 )
2294 .await;
2295 })
@@ -3121,8 +3113,7 @@ async fn non_stream_completions_include_logprobs() {
3113 rmp_serde::from_slice(&add[1]).expect("decode request");
3114 send_outputs(
3115 push,
3124 - EngineCoreOutputs {
3125 - engine_index: 0,
3116 + RequestBatchOutputs {
3117 outputs: vec![
3118 request_output_with_logprobs(
3119 &request.request_id,
@@ -3141,13 +3132,10 @@ async fn non_stream_completions_include_logprobs() {
3132 None,
3133 ),
3134 ],
3144 - scheduler_stats: None,
3145 - timestamp: 0.0,
3146 - utility_output: None,
3135 finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
3148 - wave_complete: None,
3149 - start_wave: None,
3150 - },
3136 + ..Default::default()
3137 + }
3138 + .into(),
3139 )
3140 .await;
3141 })
@@ -3224,8 +3212,7 @@ async fn non_stream_completions_include_prompt_logprobs() {
3212 rmp_serde::from_slice(&add[1]).expect("decode request");
3213 send_outputs(
3214 push,
3227 - EngineCoreOutputs {
3228 - engine_index: 0,
3215 + RequestBatchOutputs {
3216 outputs: vec![request_output_with_logprobs(
3217 &request.request_id,
3218 vec![b'h' as u32, b'i' as u32, b'!' as u32],
@@ -3246,13 +3233,9 @@ async fn non_stream_completions_include_prompt_logprobs() {
3233 }),
3234 Some(prompt_logprobs_for_hello()),
3235 )],
3249 - scheduler_stats: None,
3250 - timestamp: 0.0,
3251 - utility_output: None,
3252 - finished_requests: None,
3253 - wave_complete: None,
3254 - start_wave: None,
3255 - },
3236 + ..Default::default()
3237 + }
3238 + .into(),
3239 )
3240 .await;
3241 })
@@ -3588,8 +3571,7 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
3571
3572 send_outputs(
3573 push,
3591 - EngineCoreOutputs {
3592 - engine_index: 0,
3574 + RequestBatchOutputs {
3575 outputs: vec![
3576 request_output_with_logprobs(
3577 &request.request_id,
@@ -3609,13 +3591,9 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
3591 Some(json!({"connector": "x"})),
3592 ),
3593 ],
3612 - scheduler_stats: None,
3613 - timestamp: 0.0,
3614 - utility_output: None,
3615 - finished_requests: None,
3616 - wave_complete: None,
3617 - start_wave: None,
3618 - },
3594 + ..Default::default()
3595 + }
3596 + .into(),
3597 )
3598 .await;
3599 })
@@ -3708,8 +3686,7 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
3686
3687 send_outputs(
3688 push,
3711 - EngineCoreOutputs {
3712 - engine_index: 0,
3689 + RequestBatchOutputs {
3690 outputs: vec![
3691 request_output_with_logprobs(
3692 &request.request_id,
@@ -3728,13 +3705,9 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
3705 None,
3706 ),
3707 ],
3731 - scheduler_stats: None,
3732 - timestamp: 0.0,
3733 - utility_output: None,
3734 - finished_requests: None,
3735 - wave_complete: None,
3736 - start_wave: None,
3737 - },
3708 + ..Default::default()
3709 + }
3710 + .into(),
3711 )
3712 .await;
3713 })
@@ -4426,8 +4399,7 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
4399
4400 send_outputs(
4401 push,
4429 - EngineCoreOutputs {
4430 - engine_index: 0,
4402 + RequestBatchOutputs {
4403 outputs: vec![
4404 request_output_with_logprobs(
4405 &request.request_id,
@@ -4446,13 +4418,9 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
4418 None,
4419 ),
4420 ],
4449 - scheduler_stats: None,
4450 - timestamp: 0.0,
4451 - utility_output: None,
4452 - finished_requests: None,
4453 - wave_complete: None,
4454 - start_wave: None,
4455 - },
4421 + ..Default::default()
4422 + }
4423 + .into(),
4424 )
4425 .await;
4426 })
@@ -4601,8 +4569,7 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
4569
4570 send_outputs(
4571 push,
4604 - EngineCoreOutputs {
4605 - engine_index: 0,
4572 + RequestBatchOutputs {
4573 outputs: vec![request_output_with_logprobs(
4574 &request.request_id,
4575 bytes_to_token_ids(b"<think>Need tool.</think>"),
@@ -4613,19 +4580,14 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
4580 ))),
4581 None,
4582 )],
4616 - scheduler_stats: None,
4617 - timestamp: 0.0,
4618 - utility_output: None,
4619 - finished_requests: None,
4620 - wave_complete: None,
4621 - start_wave: None,
4622 - },
4583 + ..Default::default()
4584 + }
4585 + .into(),
4586 )
4587 .await;
4588 send_outputs(
4589 push,
4627 - EngineCoreOutputs {
4628 - engine_index: 0,
4590 + RequestBatchOutputs {
4591 outputs: vec![request_output_with_logprobs(
4592 &request.request_id,
4593 bytes_to_token_ids(b"<tool_call>\n{\"name\":\"get_weather\", "),
@@ -4636,19 +4598,14 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
4598 ))),
4599 None,
4600 )],
4639 - scheduler_stats: None,
4640 - timestamp: 0.0,
4641 - utility_output: None,
4642 - finished_requests: None,
4643 - wave_complete: None,
4644 - start_wave: None,
4645 - },
4601 + ..Default::default()
4602 + }
4603 + .into(),
4604 )
4605 .await;
4606 send_outputs(
4607 push,
4650 - EngineCoreOutputs {
4651 - engine_index: 0,
4608 + RequestBatchOutputs {
4609 outputs: vec![request_output_with_logprobs(
4610 &request.request_id,
4611 bytes_to_token_ids(
@@ -4661,13 +4618,10 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
4618 ))),
4619 None,
4620 )],
4664 - scheduler_stats: None,
4665 - timestamp: 0.0,
4666 - utility_output: None,
4621 finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
4668 - wave_complete: None,
4669 - start_wave: None,
4670 - },
4622 + ..Default::default()
4623 + }
4624 + .into(),
4625 )
4626 .await;
4627 })