@tduncan / Nvidia-Optimization-Stack-VLLM / commits / c638f9216a

[Rust Frontend] Split engine core DTOs into separate modules (#47265)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jul 1, 2026 at 22:28 UTC c638f9216a08bfb5644d8a266ddd35421e04118d
54 files changed +806 -789
rust/src/chat/src/lib.rs
+2 -1
@@ -50,7 +50,8 @@ mod request;
50 mod stream;
51
52 use vllm_engine_core_client::EngineCoreClient;
53 -use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};
53 +use vllm_engine_core_client::protocol::dtype::ModelDtype;
54 +use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
55 use vllm_llm::Llm;
56 use vllm_text::{Prompt, TextLlm, TextRequest};
57
rust/src/chat/src/multimodal.rs
+1 -1
@@ -22,7 +22,7 @@ use llm_multimodal::{
22 TrackedMedia,
23 };
24 use tracing::warn;
25 -use vllm_engine_core_client::protocol::ModelDtype;
25 +use vllm_engine_core_client::protocol::dtype::ModelDtype;
26 use vllm_engine_core_client::protocol::multimodal::{
27 MmBatchedField, MmFeatureSpec, MmFeatures, MmField, MmFieldElem, MmFlatField, MmKwargsItem,
28 MmSharedField, MmSlice, PlaceholderRange, SliceSpec,
rust/src/chat/src/multimodal/tensor.rs
+1 -1
@@ -2,7 +2,7 @@ use std::collections::HashMap;
2
3 use half::{bf16, f16};
4 use llm_multimodal::{ModelSpecificValue, PreprocessedImages};
5 -use vllm_engine_core_client::protocol::ModelDtype;
5 +use vllm_engine_core_client::protocol::dtype::ModelDtype;
6 use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
7 use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor};
8
rust/src/chat/src/output/default/structural_tag.rs
+6 -2
@@ -1,7 +1,9 @@
1 //! Applies xgrammar structural-tag constraints for strict tool calling.
2
3 use thiserror_ext::AsReport;
4 -use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
4 +use vllm_engine_core_client::protocol::structured_outputs::{
5 + StructuredOutputBackend, StructuredOutputsParams,
6 +};
7 use vllm_parser::tool::StructuralTagModel;
8 use xgrammar_structural_tag::{
9 FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
@@ -76,7 +78,9 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagTool
78 #[cfg(test)]
79 mod tests {
80 use serde_json::{Value, json};
79 - use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
81 + use vllm_engine_core_client::protocol::structured_outputs::{
82 + StructuredOutputBackend, StructuredOutputsParams,
83 + };
84 use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
85
86 use super::*;
rust/src/chat/tests/chat.rs
+3 -2
@@ -15,9 +15,10 @@ use vllm_chat::{
15 use vllm_engine_core_client::protocol::logprobs::{
16 Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
17 };
18 -use vllm_engine_core_client::protocol::{
19 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason,
18 +use vllm_engine_core_client::protocol::output::{
19 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, StopReason,
20 };
21 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
22 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
23 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
24 use vllm_llm::Llm;
rust/src/engine-core-client/examples/external_engine_logprobs.rs
+3 -3
@@ -5,9 +5,9 @@ use clap::Parser;
5 use futures::StreamExt as _;
6 use tokio::time::timeout;
7 use tracing_subscriber::EnvFilter;
8 -use vllm_engine_core_client::protocol::{
9 - EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
10 -};
8 +use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
9 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
10 +use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
11 use vllm_engine_core_client::{
12 EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
13 };
rust/src/engine-core-client/src/client.rs
+2 -1
@@ -11,10 +11,11 @@ use tracing::{debug, info, trace};
11 use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
12 use crate::coordinator::CoordinatorHandle;
13 use crate::error::{Error, Result};
14 +use crate::protocol::dtype::ModelDtype;
15 use crate::protocol::handshake::EngineCoreReadyResponse;
16 use crate::protocol::lora::LoraRequest;
17 +use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
18 use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
17 -use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
19 use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
20 use crate::transport::{self, ConnectedEngine};
21
rust/src/engine-core-client/src/client/imp.rs
+3 -4
@@ -16,12 +16,11 @@ use crate::client::stream::EngineCoreStreamOutput;
16 use crate::client::{AbortCause, AbortRequest};
17 use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
18 use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
19 +use crate::protocol::encode_msgpack;
20 +use crate::protocol::output::{ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs};
21 +use crate::protocol::request::EngineCoreRequestType;
22 use crate::protocol::stats::SchedulerStats;
23 use crate::protocol::utility::UtilityOutput;
21 -use crate::protocol::{
22 - ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
23 - encode_msgpack,
24 -};
24 use crate::transport::{ConnectedEngine, EngineId};
25 use crate::{Error, Result, transport};
26
rust/src/engine-core-client/src/client/state.rs
+2 -2
@@ -7,9 +7,9 @@ use tracing::trace;
7 use crate::EngineId;
8 use crate::client::stream::EngineCoreStreamOutput;
9 use crate::error::{Error, Result};
10 +use crate::protocol::output::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
11 use crate::protocol::stats::SchedulerStats;
12 use crate::protocol::utility::UtilityOutput;
12 -use crate::protocol::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
13 use crate::transport::ConnectedEngine;
14
15 pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
@@ -452,7 +452,7 @@ mod tests {
452 EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
453 };
454 use crate::mock_engine::default_ready_response;
455 - use crate::protocol::{
455 + use crate::protocol::output::{
456 EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
457 };
458 use crate::transport::ConnectedEngine;
rust/src/engine-core-client/src/client/stream.rs
+1 -1
@@ -10,7 +10,7 @@ use tracing::{debug, error, warn};
10
11 use crate::client::AbortRequest;
12 use crate::client::state::OutputReceiver;
13 -use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
13 +use crate::protocol::output::{EngineCoreFinishReason, EngineCoreOutput};
14 use crate::{AbortCause, Error, Result};
15
16 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
rust/src/engine-core-client/src/coordinator/inproc.rs
+3 -4
@@ -10,10 +10,9 @@ use zeromq::{XPubSocket, ZmqMessage};
10 use crate::client::imp::ClientInner;
11 use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
12 use crate::error::{Error, Result, bail_unexpected_coordinator_output};
13 -use crate::protocol::{
14 - ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs, EngineCoreRequestType,
15 - encode_msgpack,
16 -};
13 +use crate::protocol::encode_msgpack;
14 +use crate::protocol::output::{ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs};
15 +use crate::protocol::request::EngineCoreRequestType;
16
17 /// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
18 /// engine-facing coordinator socket.
rust/src/engine-core-client/src/mock_engine.rs
+2 -1
@@ -8,8 +8,9 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
8
9 use crate::EngineId;
10 use crate::error::{Error, Result, bail_unexpected_handshake_message};
11 +use crate::protocol::dtype::ModelDtype;
12 use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
12 -use crate::protocol::{ModelDtype, decode_msgpack, encode_msgpack};
13 +use crate::protocol::{decode_msgpack, encode_msgpack};
14
15 /// Default model length advertised by reusable mock engine helpers.
16 pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;
rust/src/engine-core-client/src/protocol/handshake.rs
+2 -1
@@ -2,7 +2,8 @@ use std::collections::BTreeMap;
2
3 use serde::{Deserialize, Serialize};
4
5 -use crate::protocol::{ModelDtype, OpaqueValue};
5 +use crate::protocol::OpaqueValue;
6 +use crate::protocol::dtype::ModelDtype;
7
8 /// Decoded engine startup-handshake payload sent on the handshake socket.
9 ///
rust/src/engine-core-client/src/protocol/logprobs.rs
+2 -47
@@ -9,8 +9,7 @@ use enum_as_inner::EnumAsInner;
9 use serde::{Deserialize, Deserializer, Serialize};
10
11 use self::wire::*;
12 -use super::{EngineCoreOutput, EngineCoreOutputs, decode_msgpack};
13 -use crate::error::{Error, Result, bail_ext_value_decode, ext_value_decode};
12 +use crate::error::{Error, Result, bail_ext_value_decode};
13 use crate::protocol::tensor::{WireArrayData, WireNdArray};
14
15 /// One token candidate and its logprob metadata for a single sequence position.
@@ -160,7 +159,7 @@ impl Serialize for MaybeWireLogprobs {
159 impl MaybeWireLogprobs {
160 /// Resolve the wire representation into decoded logprobs by looking up aux
161 /// frames and decoding raw views as needed.
163 - fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
162 + pub(super) fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
163 where
164 Frame: AsRef<[u8]>,
165 {
@@ -171,37 +170,6 @@ impl MaybeWireLogprobs {
170 }
171 }
172
174 -impl EngineCoreOutputs {
175 - /// Resolve all wire-format fields in-place by looking up aux frames and
176 - /// decoding raw-view payloads as needed.
177 - fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
178 - where
179 - Frame: AsRef<[u8]>,
180 - {
181 - for output in &mut self.outputs {
182 - output.resolve_in_place(frames)?;
183 - }
184 - Ok(())
185 - }
186 -}
187 -
188 -impl EngineCoreOutput {
189 - /// Resolve all wire-format fields in-place by looking up aux frames and
190 - /// decoding raw-view payloads as needed.
191 - fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
192 - where
193 - Frame: AsRef<[u8]>,
194 - {
195 - self.new_logprobs = (self.new_logprobs.take())
196 - .map(|value| value.resolve(frames, "new_logprobs"))
197 - .transpose()?;
198 - self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
199 - .map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
200 - .transpose()?;
201 - Ok(())
202 - }
203 -}
204 -
173 impl WireLogprobs {
174 /// Convert semantic per-position logprobs into the Python wire tuple shape.
175 ///
@@ -315,16 +283,3 @@ impl WireLogprobs {
283 Ok(Logprobs { positions })
284 }
285 }
318 -
319 -/// Decode one ordinary or multipart engine-core output message into the strong
320 -/// typed public protocol shape.
321 -pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
322 -where
323 - Frame: AsRef<[u8]>,
324 -{
325 - let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
326 -
327 - let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
328 - outputs.resolve_in_place(frames)?;
329 - Ok(outputs)
330 -}
rust/src/engine-core-client/src/protocol/logprobs/tests.rs
+2 -2
@@ -3,8 +3,8 @@ use std::collections::BTreeSet;
3 use bytes::Bytes;
4 use rmpv::Value;
5
6 -use super::{Logprobs, PositionLogprobs, TokenLogprob, decode_engine_core_outputs};
7 -use crate::protocol::EngineCoreFinishReason;
6 +use super::{Logprobs, PositionLogprobs, TokenLogprob};
7 +use crate::protocol::output::{EngineCoreFinishReason, decode_engine_core_outputs};
8
9 fn encode_value(value: &Value) -> Vec<u8> {
10 let mut out = Vec::new();
rust/src/engine-core-client/src/protocol/mod.rs
+6 -636
@@ -1,28 +1,11 @@
1 use std::any::type_name;
2 -use std::collections::{BTreeMap, BTreeSet, HashMap};
2 use std::io::Cursor;
3
5 -use bytes::Bytes;
4 use rmpv::Value;
5 use serde::{Deserialize, Serialize};
8 -use serde_default::DefaultFromSerde;
9 -use serde_repr::{Deserialize_repr, Serialize_repr};
10 -use serde_tuple::{Deserialize_tuple, Serialize_tuple};
6 use thiserror_ext::AsReport;
7
8 use crate::error::{Error, Result};
14 -use crate::protocol::logprobs::MaybeWireLogprobs;
15 -use crate::protocol::multimodal::MmFeatures;
16 -use crate::protocol::stats::{PrefillStats, SchedulerStats};
17 -use crate::protocol::utility::UtilityOutput;
18 -
19 -// TODO: This module currently mixes reusable frontend-facing semantic types
20 -// (for example `FinishReason`, `StopReason`, `RequestOutputKind`, and future
21 -// cleaned-up frontend sampling types) with engine-core-specific wire DTOs and
22 -// handshake/control messages. While the Rust frontend is still evolving
23 -// quickly, keep them co-located here for iteration speed. Once the higher-level
24 -// API boundary stabilizes, move the truly reusable semantic types into a
25 -// lower-level common crate and keep the engine transport/wire messages here.
9
10 /// Dynamic msgpack value used for schema positions that are preserved but not
11 /// yet strongly typed in the early-stage Rust client.
@@ -36,499 +19,18 @@ fn is_false(v: &bool) -> bool {
19 !v
20 }
21
39 -fn default_top_p() -> f32 {
40 - 1.0
41 -}
42 -
43 -fn default_repetition_penalty() -> f32 {
44 - 1.0
45 -}
46 -
47 -fn default_temperature() -> f32 {
48 - 1.0
49 -}
50 -
51 -fn default_max_tokens() -> u32 {
52 - 16
53 -}
54 -
55 -mod classified_outputs;
22 pub mod dtype;
23 pub mod handshake;
24 pub mod logprobs;
25 pub mod lora;
26 pub mod multimodal;
27 +pub mod output;
28 +pub mod request;
29 +pub mod sampling;
30 pub mod stats;
31 +pub mod structured_outputs;
32 pub mod tensor;
33 pub mod utility;
64 -pub use classified_outputs::{
65 - ClassifiedEngineCoreOutputs, DpControlMessage, RequestBatchOutputs, UtilityCallOutput,
66 -};
67 -pub use dtype::ModelDtype;
68 -pub use logprobs::decode_engine_core_outputs;
69 -
70 -/// Request types are encoded as single-byte protocol constants so they can be
71 -/// sent over the ZMQ socket without an extra encoding step.
72 -///
73 -/// Original Python definition:
74 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
75 -#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76 -#[repr(u8)]
77 -pub enum EngineCoreRequestType {
78 - Add = 0,
79 - Abort = 1,
80 - StartDpWave = 2,
81 - Utility = 3,
82 -}
83 -
84 -impl EngineCoreRequestType {
85 - /// Decode the single-byte request type frame used on the engine input
86 - /// socket. Returns `None` for unrecognized values.
87 - pub fn from_frame(frame: &[u8]) -> Option<Self> {
88 - let [value] = frame else {
89 - return None;
90 - };
91 -
92 - match value {
93 - 0 => Some(Self::Add),
94 - 1 => Some(Self::Abort),
95 - 2 => Some(Self::StartDpWave),
96 - 3 => Some(Self::Utility),
97 - _ => None,
98 - }
99 - }
100 -
101 - /// Encode the request type as the single-byte frame used on the engine
102 - /// input socket.
103 - pub fn to_frame(self) -> Bytes {
104 - Bytes::from_static(match self {
105 - Self::Add => b"\x00",
106 - Self::Abort => b"\x01",
107 - Self::StartDpWave => b"\x02",
108 - Self::Utility => b"\x03",
109 - })
110 - }
111 -}
112 -
113 -/// Reason a request finished: stop, length, abort, error, or repetition.
114 -///
115 -/// This mirrors the Python enum and uses integer encoding for compact wire
116 -/// representation.
117 -///
118 -/// Original Python definition:
119 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
120 -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
121 -#[repr(u8)]
122 -pub enum EngineCoreFinishReason {
123 - /// A stop string was emitted.
124 - Stop = 0,
125 - /// `max_tokens` or `max_model_len` was reached.
126 - Length = 1,
127 - /// The request was aborted by the client.
128 - Abort = 2,
129 - /// A retryable request-level internal error occurred.
130 - Error = 3,
131 - /// A repetitive token pattern was detected.
132 - Repetition = 4,
133 -}
134 -
135 -/// Event types emitted by engine-core for one request.
136 -///
137 -/// Original Python definition:
138 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
139 -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
140 -#[repr(u8)]
141 -pub enum EngineCoreEventType {
142 - Queued = 1,
143 - Scheduled = 2,
144 - Preempted = 3,
145 -}
146 -
147 -/// A timestamped engine-core event associated with one request.
148 -///
149 -/// Original Python definition:
150 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
151 -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
152 -pub struct EngineCoreEvent {
153 - pub r#type: EngineCoreEventType,
154 - pub timestamp: f64,
155 -}
156 -
157 -/// Controls how intermediate outputs are returned to the frontend.
158 -///
159 -/// `Cumulative = 0` is intentionally not supported in Rust frontend.
160 -///
161 -/// Original Python definition:
162 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L146-L152>
163 -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize_repr, Deserialize_repr)]
164 -#[repr(u8)]
165 -pub enum RequestOutputKind {
166 - /// Return only token deltas in each update.
167 - #[default]
168 - Delta = 1,
169 - /// Suppress intermediate updates and return only the final output.
170 - FinalOnly = 2,
171 -}
172 -
173 -/// Structured-output backend selected for EngineCore grammar compilation.
174 -///
175 -/// Python vLLM stores this in `StructuredOutputsParams._backend` after request
176 -/// validation. The Rust frontend currently always lowers structured-output
177 -/// requests to guidance, while ignoring any user-supplied `_backend` value.
178 -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
179 -#[serde(rename_all = "kebab-case")]
180 -pub enum StructuredOutputBackend {
181 - Xgrammar,
182 - #[default]
183 - Guidance,
184 - Outlines,
185 - LmFormatEnforcer,
186 -}
187 -
188 -/// The stop reason associated with a finished output.
189 -///
190 -/// Python models this as the union-typed `stop_reason: int | str | None`
191 -/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
192 -///
193 -/// Original Python field:
194 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
195 -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
196 -#[serde(untagged)]
197 -pub enum StopReason {
198 - TokenId(u32),
199 - Text(String),
200 -}
201 -
202 -/// Parameters for configuring structured outputs (guided decoding).
203 -///
204 -/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
205 -/// `json_object`, or `structural_tag`) should be set. The engine-core
206 -/// backend selects the appropriate grammar compiler based on which field
207 -/// is present.
208 -///
209 -/// Original Python definition:
210 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
211 -#[serde_with::skip_serializing_none]
212 -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
213 -#[serde(default)]
214 -pub struct StructuredOutputsParams {
215 - /// JSON schema (as a dict/object or JSON string) constraining the output.
216 - pub json: Option<serde_json::Value>,
217 - /// Regular expression the output must match.
218 - pub regex: Option<String>,
219 - /// List of allowed output strings (the model must produce one of these).
220 - pub choice: Option<Vec<String>>,
221 - /// Context-free grammar (in EBNF-like notation) the output must conform to.
222 - pub grammar: Option<String>,
223 - /// When `true`, output must be valid JSON (free-form, no schema).
224 - pub json_object: Option<bool>,
225 - /// Disable any additional whitespace in guided JSON output.
226 - #[serde(skip_serializing_if = "crate::protocol::is_false")]
227 - pub disable_any_whitespace: bool,
228 - /// Disable `additionalProperties` in JSON schema output.
229 - #[serde(skip_serializing_if = "crate::protocol::is_false")]
230 - pub disable_additional_properties: bool,
231 - /// Custom whitespace pattern for guided JSON output.
232 - pub whitespace_pattern: Option<String>,
233 - /// Structural tag configuration (JSON-encoded string).
234 - pub structural_tag: Option<String>,
235 - /// Structured-output backend, mirroring Python's internal `_backend`.
236 - ///
237 - /// User-supplied values are ignored during deserialization. This matches
238 - /// Python's request boundary, where `_backend` is set by validation rather
239 - /// than accepted as a request-level backend selector.
240 - #[serde(
241 - default,
242 - rename = "_backend",
243 - deserialize_with = "serde_with::rust::deserialize_ignore_any"
244 - )]
245 - pub backend: StructuredOutputBackend,
246 -}
247 -
248 -/// Engine-core-facing sampling parameters for text generation.
249 -///
250 -/// This is the normalized southbound subset used by the Rust frontend when it
251 -/// talks to Python engine-core over the wire. User-facing request semantics
252 -/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
253 -/// intentionally handled by higher layers before values reach this DTO.
254 -///
255 -/// Original Python definition:
256 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
257 -// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
258 -// default-valued keys; default the whole struct. Per-field fns cover the
259 -// non-zero defaults.
260 -#[serde_with::skip_serializing_none]
261 -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
262 -#[serde(default)]
263 -pub struct EngineCoreSamplingParams {
264 - /// Controls randomness. Lower values are more deterministic; zero means
265 - /// greedy sampling.
266 - #[serde(default = "default_temperature")]
267 - pub temperature: f32,
268 - /// Cumulative probability threshold for nucleus sampling.
269 - #[serde(default = "default_top_p")]
270 - pub top_p: f32,
271 - /// Maximum number of top tokens to consider. `0` means all tokens.
272 - pub top_k: u32,
273 - /// Random seed used by the sampler when present.
274 - pub seed: Option<i64>,
275 - /// Maximum number of tokens to generate per output sequence.
276 - #[serde(default = "default_max_tokens")]
277 - pub max_tokens: u32,
278 - /// Minimum number of tokens to generate before EOS or stop-token handling.
279 - pub min_tokens: u32,
280 - /// Maximum number of reasoning ("thinking") tokens to emit before the
281 - /// reasoning section is force-closed. `None` means unlimited; the
282 - /// user-facing `-1` sentinel is normalized to `None` by the frontend before
283 - /// reaching this DTO, so only non-negative values are sent. Enforced
284 - /// engine-side (and only when a reasoning parser is configured).
285 - pub thinking_token_budget: Option<u64>,
286 - /// Number of log probabilities to return per generated token.
287 - ///
288 - /// `None` disables sample logprobs. `-1` requests the full vocabulary.
289 - pub logprobs: Option<i32>,
290 - /// Number of log probabilities to return per prompt token.
291 - ///
292 - /// `None` disables prompt logprobs. `-1` requests the full vocabulary.
293 - pub prompt_logprobs: Option<i32>,
294 - /// Minimum probability threshold for token sampling.
295 - pub min_p: f32,
296 - /// Frequency penalty applied by the sampler.
297 - pub frequency_penalty: f32,
298 - /// Presence penalty applied by the sampler.
299 - pub presence_penalty: f32,
300 - /// Repetition penalty applied by the sampler.
301 - #[serde(default = "default_repetition_penalty")]
302 - pub repetition_penalty: f32,
303 - /// Token IDs that stop generation.
304 - pub stop_token_ids: Vec<u32>,
305 - /// Primary EOS token ID used by engine-core's dedicated EOS stop path.
306 - ///
307 - /// This mirrors Python's internal `_eos_token_id` field and is derived by
308 - /// the frontend from tokenizer/model metadata rather than supplied directly
309 - /// by end users.
310 - #[serde(rename = "_eos_token_id")]
311 - pub eos_token_id: Option<u32>,
312 - /// Complete stop-token set used by engine-core for `min_tokens` masking.
313 - ///
314 - /// This mirrors Python's internal `_all_stop_token_ids` field and should
315 - /// contain explicit `stop_token_ids` plus any frontend-derived EOS token
316 - /// IDs.
317 - #[serde(rename = "_all_stop_token_ids")]
318 - pub all_stop_token_ids: BTreeSet<u32>,
319 - /// Logit biases to apply during sampling.
320 - /// Keys are token IDs
321 - pub logit_bias: Option<HashMap<u32, f32>>,
322 - /// Restrict output to these token IDs only.
323 - pub allowed_token_ids: Option<Vec<u32>>,
324 - /// Tokenized bad words to avoid during generation.
325 - #[serde(rename = "_bad_words_token_ids")]
326 - pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
327 - /// Parameters for configuring structured outputs (guided decoding).
328 - pub structured_outputs: Option<StructuredOutputsParams>,
329 - /// Specific token IDs for which log probabilities should be returned at
330 - /// each position.
331 - ///
332 - /// When set, the engine returns logprobs for exactly these tokens in
333 - /// addition to the sampled/scored token. Mutually exclusive with the
334 - /// `logprobs` count field in practice.
335 - pub logprob_token_ids: Option<Vec<u32>>,
336 - /// If `Some(true)`, the request will not attempt to read from the prefix
337 - /// cache; newly computed blocks may still populate the cache. `None`
338 - /// defers to engine-core defaults.
339 - pub skip_reading_prefix_cache: Option<bool>,
340 - /// Additional request parameters for custom extensions (from `vllm_xargs`).
341 - pub extra_args: Option<HashMap<String, serde_json::Value>>,
342 -}
343 -
344 -impl EngineCoreSamplingParams {
345 - /// Constructs a default sampling params for testing purposes only.
346 - pub fn for_test() -> Self {
347 - Self {
348 - temperature: 1.0,
349 - top_p: 1.0,
350 - top_k: 0,
351 - seed: None,
352 - max_tokens: 65536,
353 - min_tokens: 0,
354 - thinking_token_budget: None,
355 - logprobs: None,
356 - prompt_logprobs: None,
357 - min_p: 0.0,
358 - frequency_penalty: 0.0,
359 - presence_penalty: 0.0,
360 - repetition_penalty: 1.0,
361 - stop_token_ids: Vec::new(),
362 - eos_token_id: None,
363 - all_stop_token_ids: BTreeSet::new(),
364 - logit_bias: None,
365 - allowed_token_ids: None,
366 - bad_words_token_ids: None,
367 - structured_outputs: None,
368 - logprob_token_ids: None,
369 - skip_reading_prefix_cache: None,
370 - extra_args: None,
371 - }
372 - }
373 -}
374 -
375 -/// Extra kwargs consumed by engine-side reasoning parsers.
376 -///
377 -/// Original Python construction point:
378 -/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
379 -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
380 -pub struct ReasoningParserKwargs {
381 - /// Effective kwargs visible to the chat template for this request.
382 - pub chat_template_kwargs: HashMap<String, serde_json::Value>,
383 -}
384 -
385 -/// Engine-core add-request payload sent from frontend to engine.
386 -///
387 -/// Original Python definition:
388 -/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
389 -#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
390 -pub struct EngineCoreRequest {
391 - pub request_id: String,
392 - pub prompt_token_ids: Option<Vec<u32>>,
393 - /// Multimodal features attached to the request.
394 - pub mm_features: Option<MmFeatures>,
395 - pub sampling_params: Option<EngineCoreSamplingParams>,
396 - /// Pooling parameters are preserved in the schema but not yet strongly
397 - /// typed.
398 - pub pooling_params: Option<OpaqueValue>,
399 - pub arrival_time: f64,
400 - #[serde(default)]
401 - pub lora_request: Option<lora::LoraRequest>,
402 - #[serde(default)]
403 - pub cache_salt: Option<String>,
404 - #[serde(default)]
405 - pub data_parallel_rank: Option<u32>,
406 - /// Unsupported in the first-stage Rust client because Python uses a custom
407 - /// tensor/aux-frame encoding path for this field.
408 - #[serde(default)]
409 - pub prompt_embeds: Option<OpaqueValue>,
410 - /// Per-position mask for mixed-mode inputs (e.g. chat completion with
411 - /// `prompt_embeds` content parts). `Some(true)` means real token id;
412 - /// `Some(false)` means the position uses a pre-computed entry from
413 - /// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
414 - #[serde(default)]
415 - pub prompt_is_token_ids: Option<Vec<bool>>,
416 - /// Index of the client, used to ensure outputs are sent back to the same
417 - /// client when scaling out the frontend.
418 - #[serde(default)]
419 - pub client_index: u32,
420 - /// In DP mode, indicates which wave this request is expected to belong to.
421 - #[serde(default)]
422 - pub current_wave: u32,
423 - #[serde(default)]
424 - pub priority: i32,
425 - #[serde(default)]
426 - pub trace_headers: Option<BTreeMap<String, String>>,
427 - #[serde(default)]
428 - pub resumable: bool,
429 - /// Original user-provided request ID, used for output reporting and aborts.
430 - #[serde(default)]
431 - pub external_req_id: Option<String>,
432 - #[serde(default)]
433 - pub reasoning_ended: Option<bool>,
434 - /// Reasoning-parser kwargs forwarded from the frontend to the
435 - /// structured-output backend.
436 - #[serde(default)]
437 - pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
438 - /// If `true`, the request should be added to the scheduler's waiting queue
439 - /// and immediately aborted, so connector-side cleanup runs via the
440 - /// standard `request_finished` hook.
441 - #[serde(default)]
442 - pub abort_immediately: bool,
443 -}
444 -
445 -impl EngineCoreRequest {
446 - /// Validate fields intentionally not supported in the first-stage client.
447 - pub fn validate(&self) -> Result<()> {
448 - if self.prompt_embeds.is_some() {
449 - return Err(Error::UnsupportedField {
450 - context: "EngineCoreRequest",
451 - field: "prompt_embeds",
452 - });
453 - }
454 - Ok(())
455 - }
456 -}
457 -
458 -/// Engine-core output for a single request.
459 -///
460 -/// Original Python definition:
461 -/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
462 -#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
463 -pub struct EngineCoreOutput {
464 - pub request_id: String,
465 - pub new_token_ids: Vec<u32>,
466 - /// Decoded sample logprobs for the newly generated positions in this
467 - /// output.
468 - #[serde(default)]
469 - pub new_logprobs: Option<MaybeWireLogprobs>,
470 - /// Decoded prompt logprobs for the scored prompt positions emitted in this
471 - /// output.
472 - #[serde(default)]
473 - pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
474 - #[serde(default)]
475 - pub pooling_output: Option<OpaqueValue>,
476 - #[serde(default)]
477 - pub finish_reason: Option<EngineCoreFinishReason>,
478 - #[serde(default)]
479 - pub stop_reason: Option<StopReason>,
480 - #[serde(default)]
481 - pub events: Option<Vec<EngineCoreEvent>>,
482 - #[serde(default)]
483 - pub kv_transfer_params: Option<serde_json::Value>,
484 - #[serde(default)]
485 - pub trace_headers: Option<OpaqueValue>,
486 - /// Breakdown of the scheduled prefill computation, set on the first output
487 - /// of a newly scheduled prefill and elided for subsequent decode outputs.
488 - #[serde(default)]
489 - pub prefill_stats: Option<PrefillStats>,
490 - #[serde(default)]
491 - pub routed_experts: Option<OpaqueValue>,
492 - /// Number of NaNs seen in logits. Values above zero indicate corruption.
493 - #[serde(default)]
494 - pub num_nans_in_logits: u32,
495 -}
496 -
497 -impl EngineCoreOutput {
498 - /// Returns whether this output is terminal for the request.
499 - pub fn finished(&self) -> bool {
500 - self.finish_reason.is_some()
501 - }
502 -}
503 -
504 -/// Batch of engine-core outputs returned to a frontend client.
505 -///
506 -/// Original Python definition:
507 -/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
508 -#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
509 -pub struct EngineCoreOutputs {
510 - #[serde(default)]
511 - pub engine_index: u32,
512 - /// Outputs grouped for this client in the current engine tick.
513 - #[serde(default)]
514 - pub outputs: Vec<EngineCoreOutput>,
515 - #[serde(default)]
516 - pub scheduler_stats: Option<Box<SchedulerStats>>,
517 - #[serde(default)]
518 - pub timestamp: f64,
519 - #[serde(default)]
520 - pub utility_output: Option<UtilityOutput>,
521 - #[serde(default)]
522 - pub finished_requests: Option<BTreeSet<String>>,
523 - /// In DP mode, signals that the current wave finished and engines are
524 - /// paused.
525 - #[serde(default)]
526 - pub wave_complete: Option<u32>,
527 - /// In DP mode, signals that a request arrived for an old wave and the next
528 - /// wave needs to start in other engines.
529 - #[serde(default)]
530 - pub start_wave: Option<u32>,
531 -}
34
35 /// Encode a Rust value into msgpack using the protocol crate's serde model.
36 pub fn encode_msgpack<T>(value: &T) -> Result<Vec<u8>>
@@ -564,81 +66,17 @@ where
66 })
67 }
68
69 +/// Decode a msgpack payload into a dynamic value for diagnostics and tests.
70 pub fn decode_value(bytes: &[u8]) -> Result<Value> {
71 Ok(rmpv::decode::read_value(&mut Cursor::new(bytes))?)
72 }
73
74 #[cfg(test)]
75 mod tests {
573 - use std::collections::BTreeSet;
76 + use std::collections::BTreeMap;
77
78 use super::*;
79
577 - #[test]
578 - fn engine_core_request_serializes_as_full_array() {
579 - let request = EngineCoreRequest {
580 - request_id: "req-1".to_string(),
581 - prompt_token_ids: Some(vec![1, 2, 3]),
582 - sampling_params: Some(EngineCoreSamplingParams {
583 - max_tokens: 8,
584 - ..EngineCoreSamplingParams::for_test()
585 - }),
586 - arrival_time: 1234.5,
587 - client_index: 7,
588 - ..EngineCoreRequest::default()
589 - };
590 -
591 - let encoded = encode_msgpack(&request).unwrap();
592 - let value = decode_value(&encoded).unwrap();
593 - let array = match value {
594 - Value::Array(array) => array,
595 - other => panic!("expected array, got {other:?}"),
596 - };
597 -
598 - assert_eq!(array.len(), 20);
599 - assert_eq!(array[0], Value::from("req-1"));
600 - assert_eq!(array[2], Value::Nil);
601 - assert_eq!(array[4], Value::Nil);
602 - assert_eq!(array[10], Value::Nil);
603 - assert_eq!(array[11], Value::from(7));
604 - }
605 -
606 - #[test]
607 - fn engine_core_outputs_roundtrip_finished_fields() {
608 - let outputs = EngineCoreOutputs {
609 - outputs: vec![EngineCoreOutput {
610 - request_id: "req-1".to_string(),
611 - new_token_ids: vec![42],
612 - new_logprobs: None,
613 - new_prompt_logprobs_tensors: None,
614 - pooling_output: None,
615 - finish_reason: Some(EngineCoreFinishReason::Length),
616 - stop_reason: Some(StopReason::Text("stop".to_string())),
617 - events: None,
618 - kv_transfer_params: None,
619 - trace_headers: None,
620 - prefill_stats: None,
621 - routed_experts: None,
622 - num_nans_in_logits: 0,
623 - }],
624 - finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
625 - ..Default::default()
626 - };
627 -
628 - let encoded = encode_msgpack(&outputs).unwrap();
629 - let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
630 -
631 - assert_eq!(decoded.outputs.len(), 1);
632 - assert_eq!(
633 - decoded.outputs[0].finish_reason,
634 - Some(EngineCoreFinishReason::Length)
635 - );
636 - assert_eq!(
637 - decoded.finished_requests,
638 - Some(BTreeSet::from(["req-1".to_string()]))
639 - );
640 - }
641 -
80 #[test]
81 fn decode_msgpack_includes_type_name_and_value_fallback() {
82 let error = decode_msgpack::<u64>(
@@ -648,72 +86,4 @@ mod tests {
86
87 expect_test::expect![[r#"messagepack decode failed for u64: wrong msgpack marker FixMap(1); value fallback: {"status": "READY"}"#]].assert_eq(&error.to_report_string());
88 }
651 -
652 - #[test]
653 - fn structured_outputs_backend_ignores_deserialized_value() {
654 - let params: StructuredOutputsParams = serde_json::from_value(serde_json::json!({
655 - "json_object": true,
656 - "_backend": "xgrammar",
657 - }))
658 - .unwrap();
659 -
660 - assert_eq!(params.backend, StructuredOutputBackend::Guidance);
661 -
662 - let value = serde_json::to_value(params).unwrap();
663 - assert_eq!(value["_backend"], "guidance");
664 - }
665 -
666 - /// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
667 - /// must fall back to defaults. `python_compat` can't catch this since Rust
668 - /// encodes full maps (see `engine_core_request_serializes_as_full_array`).
669 - #[test]
670 - fn decodes_sampling_params_with_omitted_defaults() {
671 - let sampling_params = Value::Map(vec![
672 - (
673 - Value::from("stop_token_ids"),
674 - Value::Array(vec![Value::from(151643u32)]),
675 - ),
676 - (Value::from("skip_reading_prefix_cache"), Value::from(false)),
677 - ]);
678 - let request = Value::Array(vec![
679 - Value::from("req-omit-defaults"),
680 - Value::Array(vec![
681 - Value::from(1u32),
682 - Value::from(2u32),
683 - Value::from(3u32),
684 - ]),
685 - Value::Nil,
686 - sampling_params,
687 - Value::Nil,
688 - Value::from(1.0f64),
689 - ]);
690 -
691 - let mut bytes = Vec::new();
692 - rmpv::encode::write_value(&mut bytes, &request).unwrap();
693 -
694 - let decoded: EngineCoreRequest = decode_msgpack(&bytes)
695 - .expect("a real omit_defaults request must decode (regression: missing field)");
696 -
697 - assert_eq!(decoded.request_id, "req-omit-defaults");
698 - let sampling = decoded.sampling_params.expect("sampling params present");
699 -
700 - assert_eq!(sampling.stop_token_ids, vec![151643]);
701 - assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
702 -
703 - // Omitted fields -> Python defaults.
704 - assert_eq!(sampling.temperature, 1.0);
705 - assert_eq!(sampling.top_p, 1.0);
706 - assert_eq!(sampling.top_k, 0);
707 - assert_eq!(sampling.seed, None);
708 - assert_eq!(sampling.max_tokens, 16);
709 - assert_eq!(sampling.min_tokens, 0);
710 - assert_eq!(sampling.min_p, 0.0);
711 - assert_eq!(sampling.frequency_penalty, 0.0);
712 - assert_eq!(sampling.presence_penalty, 0.0);
713 - assert_eq!(sampling.repetition_penalty, 1.0);
714 - assert_eq!(sampling.logprobs, None);
715 - assert_eq!(sampling.prompt_logprobs, None);
716 - assert_eq!(sampling.eos_token_id, None);
717 - assert!(sampling.all_stop_token_ids.is_empty());
718 - }
89 }
rust/src/engine-core-client/src/protocol/output.rs renamed
+219 -3
@@ -1,10 +1,164 @@
1 use std::collections::BTreeSet;
2
3 use enum_as_inner::EnumAsInner;
4 +use serde::{Deserialize, Serialize};
5 +use serde_default::DefaultFromSerde;
6 +use serde_repr::{Deserialize_repr, Serialize_repr};
7 +use serde_tuple::{Deserialize_tuple, Serialize_tuple};
8
9 use super::utility::UtilityOutput;
6 -use super::{EngineCoreOutput, EngineCoreOutputs};
7 -use crate::protocol::stats::SchedulerStats;
10 +use crate::error::{Error, Result, ext_value_decode};
11 +use crate::protocol::logprobs::MaybeWireLogprobs;
12 +use crate::protocol::stats::{PrefillStats, SchedulerStats};
13 +use crate::protocol::{OpaqueValue, decode_msgpack};
14 +
15 +/// The stop reason associated with a finished output.
16 +///
17 +/// Python models this as the union-typed `stop_reason: int | str | None`
18 +/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
19 +///
20 +/// Original Python field:
21 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
22 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
23 +#[serde(untagged)]
24 +pub enum StopReason {
25 + TokenId(u32),
26 + Text(String),
27 +}
28 +
29 +/// Reason a request finished: stop, length, abort, error, or repetition.
30 +///
31 +/// This mirrors the Python enum and uses integer encoding for compact wire
32 +/// representation.
33 +///
34 +/// Original Python definition:
35 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
36 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
37 +#[repr(u8)]
38 +pub enum EngineCoreFinishReason {
39 + /// A stop string was emitted.
40 + Stop = 0,
41 + /// `max_tokens` or `max_model_len` was reached.
42 + Length = 1,
43 + /// The request was aborted by the client.
44 + Abort = 2,
45 + /// A retryable request-level internal error occurred.
46 + Error = 3,
47 + /// A repetitive token pattern was detected.
48 + Repetition = 4,
49 +}
50 +
51 +/// Event types emitted by engine-core for one request.
52 +///
53 +/// Original Python definition:
54 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
55 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
56 +#[repr(u8)]
57 +pub enum EngineCoreEventType {
58 + Queued = 1,
59 + Scheduled = 2,
60 + Preempted = 3,
61 +}
62 +
63 +/// A timestamped engine-core event associated with one request.
64 +///
65 +/// Original Python definition:
66 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
67 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68 +pub struct EngineCoreEvent {
69 + pub r#type: EngineCoreEventType,
70 + pub timestamp: f64,
71 +}
72 +
73 +/// Engine-core output for a single request.
74 +///
75 +/// Original Python definition:
76 +/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
77 +#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
78 +pub struct EngineCoreOutput {
79 + pub request_id: String,
80 + pub new_token_ids: Vec<u32>,
81 + /// Decoded sample logprobs for the newly generated positions in this
82 + /// output.
83 + #[serde(default)]
84 + pub new_logprobs: Option<MaybeWireLogprobs>,
85 + /// Decoded prompt logprobs for the scored prompt positions emitted in this
86 + /// output.
87 + #[serde(default)]
88 + pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
89 + #[serde(default)]
90 + pub pooling_output: Option<OpaqueValue>,
91 + #[serde(default)]
92 + pub finish_reason: Option<EngineCoreFinishReason>,
93 + #[serde(default)]
94 + pub stop_reason: Option<StopReason>,
95 + #[serde(default)]
96 + pub events: Option<Vec<EngineCoreEvent>>,
97 + #[serde(default)]
98 + pub kv_transfer_params: Option<serde_json::Value>,
99 + #[serde(default)]
100 + pub trace_headers: Option<OpaqueValue>,
101 + /// Breakdown of the scheduled prefill computation, set on the first output
102 + /// of a newly scheduled prefill and elided for subsequent decode outputs.
103 + #[serde(default)]
104 + pub prefill_stats: Option<PrefillStats>,
105 + #[serde(default)]
106 + pub routed_experts: Option<OpaqueValue>,
107 + /// Number of NaNs seen in logits. Values above zero indicate corruption.
108 + #[serde(default)]
109 + pub num_nans_in_logits: u32,
110 +}
111 +
112 +impl EngineCoreOutput {
113 + /// Returns whether this output is terminal for the request.
114 + pub fn finished(&self) -> bool {
115 + self.finish_reason.is_some()
116 + }
117 +
118 + /// Resolve all wire-format fields in-place by looking up aux frames and
119 + /// decoding raw-view payloads as needed.
120 + fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
121 + where
122 + Frame: AsRef<[u8]>,
123 + {
124 + self.new_logprobs = (self.new_logprobs.take())
125 + .map(|value| value.resolve(frames, "new_logprobs"))
126 + .transpose()?;
127 + self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
128 + .map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
129 + .transpose()?;
130 + Ok(())
131 + }
132 +}
133 +
134 +/// Batch of engine-core outputs returned to a frontend client.
135 +///
136 +/// Original Python definition:
137 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
138 +#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
139 +pub struct EngineCoreOutputs {
140 + #[serde(default)]
141 + pub engine_index: u32,
142 + /// Outputs grouped for this client in the current engine tick.
143 + #[serde(default)]
144 + pub outputs: Vec<EngineCoreOutput>,
145 + #[serde(default)]
146 + pub scheduler_stats: Option<Box<SchedulerStats>>,
147 + #[serde(default)]
148 + pub timestamp: f64,
149 + #[serde(default)]
150 + pub utility_output: Option<UtilityOutput>,
151 + #[serde(default)]
152 + pub finished_requests: Option<BTreeSet<String>>,
153 + /// In DP mode, signals that the current wave finished and engines are
154 + /// paused.
155 + #[serde(default)]
156 + pub wave_complete: Option<u32>,
157 + /// In DP mode, signals that a request arrived for an old wave and the next
158 + /// wave needs to start in other engines.
159 + #[serde(default)]
160 + pub start_wave: Option<u32>,
161 +}
162
163 /// Data-parallel control notifications multiplexed through `EngineCoreOutputs`.
164 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -49,6 +203,18 @@ pub enum ClassifiedEngineCoreOutputs {
203 }
204
205 impl EngineCoreOutputs {
206 + /// Resolve all wire-format fields in-place by looking up aux frames and
207 + /// decoding raw-view payloads as needed.
208 + fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
209 + where
210 + Frame: AsRef<[u8]>,
211 + {
212 + for output in &mut self.outputs {
213 + output.resolve_in_place(frames)?;
214 + }
215 + Ok(())
216 + }
217 +
218 /// Classify the raw wire message into a more semantic Rust enum.
219 pub fn classify(self) -> ClassifiedEngineCoreOutputs {
220 let has_request_payload = !self.outputs.is_empty()
@@ -92,12 +258,62 @@ impl EngineCoreOutputs {
258 }
259 }
260
261 +/// Decode one ordinary or multipart engine-core output message into the strong
262 +/// typed public protocol shape.
263 +pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
264 +where
265 + Frame: AsRef<[u8]>,
266 +{
267 + let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
268 +
269 + let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
270 + outputs.resolve_in_place(frames)?;
271 + Ok(outputs)
272 +}
273 +
274 #[cfg(test)]
275 mod tests {
276 use std::collections::BTreeSet;
277
278 use super::*;
100 - use crate::protocol::EngineCoreOutput;
279 + use crate::protocol::output::EngineCoreOutput;
280 + use crate::protocol::{decode_msgpack, encode_msgpack};
281 +
282 + #[test]
283 + fn engine_core_outputs_roundtrip_finished_fields() {
284 + let outputs = EngineCoreOutputs {
285 + outputs: vec![EngineCoreOutput {
286 + request_id: "req-1".to_string(),
287 + new_token_ids: vec![42],
288 + new_logprobs: None,
289 + new_prompt_logprobs_tensors: None,
290 + pooling_output: None,
291 + finish_reason: Some(EngineCoreFinishReason::Length),
292 + stop_reason: Some(StopReason::Text("stop".to_string())),
293 + events: None,
294 + kv_transfer_params: None,
295 + trace_headers: None,
296 + prefill_stats: None,
297 + routed_experts: None,
298 + num_nans_in_logits: 0,
299 + }],
300 + finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
301 + ..Default::default()
302 + };
303 +
304 + let encoded = encode_msgpack(&outputs).unwrap();
305 + let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
306 +
307 + assert_eq!(decoded.outputs.len(), 1);
308 + assert_eq!(
309 + decoded.outputs[0].finish_reason,
310 + Some(EngineCoreFinishReason::Length)
311 + );
312 + assert_eq!(
313 + decoded.finished_requests,
314 + Some(BTreeSet::from(["req-1".to_string()]))
315 + );
316 + }
317
318 #[test]
319 fn engine_core_outputs_classify_request_batch() {
rust/src/engine-core-client/src/protocol/request.rs new
+175
@@ -0,0 +1,175 @@
1 +use std::collections::{BTreeMap, HashMap};
2 +
3 +use bytes::Bytes;
4 +use serde::{Deserialize, Serialize};
5 +use serde_default::DefaultFromSerde;
6 +use serde_tuple::{Deserialize_tuple, Serialize_tuple};
7 +
8 +use crate::protocol::multimodal::MmFeatures;
9 +use crate::protocol::sampling::EngineCoreSamplingParams;
10 +use crate::protocol::{OpaqueValue, lora};
11 +use crate::{Error, Result};
12 +
13 +/// Request types are encoded as single-byte protocol constants so they can be
14 +/// sent over the ZMQ socket without an extra encoding step.
15 +///
16 +/// Original Python definition:
17 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
18 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19 +#[repr(u8)]
20 +pub enum EngineCoreRequestType {
21 + Add = 0,
22 + Abort = 1,
23 + StartDpWave = 2,
24 + Utility = 3,
25 +}
26 +
27 +impl EngineCoreRequestType {
28 + /// Decode the single-byte request type frame used on the engine input
29 + /// socket. Returns `None` for unrecognized values.
30 + pub fn from_frame(frame: &[u8]) -> Option<Self> {
31 + let [value] = frame else {
32 + return None;
33 + };
34 +
35 + match value {
36 + 0 => Some(Self::Add),
37 + 1 => Some(Self::Abort),
38 + 2 => Some(Self::StartDpWave),
39 + 3 => Some(Self::Utility),
40 + _ => None,
41 + }
42 + }
43 +
44 + /// Encode the request type as the single-byte frame used on the engine
45 + /// input socket.
46 + pub fn to_frame(self) -> Bytes {
47 + Bytes::from_static(match self {
48 + Self::Add => b"\x00",
49 + Self::Abort => b"\x01",
50 + Self::StartDpWave => b"\x02",
51 + Self::Utility => b"\x03",
52 + })
53 + }
54 +}
55 +
56 +/// Extra kwargs consumed by engine-side reasoning parsers.
57 +///
58 +/// Original Python construction point:
59 +/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
60 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
61 +pub struct ReasoningParserKwargs {
62 + /// Effective kwargs visible to the chat template for this request.
63 + pub chat_template_kwargs: HashMap<String, serde_json::Value>,
64 +}
65 +
66 +/// Engine-core add-request payload sent from frontend to engine.
67 +///
68 +/// Original Python definition:
69 +/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
70 +#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
71 +pub struct EngineCoreRequest {
72 + pub request_id: String,
73 + pub prompt_token_ids: Option<Vec<u32>>,
74 + /// Multimodal features attached to the request.
75 + pub mm_features: Option<MmFeatures>,
76 + pub sampling_params: Option<EngineCoreSamplingParams>,
77 + /// Pooling parameters are preserved in the schema but not yet strongly
78 + /// typed.
79 + pub pooling_params: Option<OpaqueValue>,
80 + pub arrival_time: f64,
81 + #[serde(default)]
82 + pub lora_request: Option<lora::LoraRequest>,
83 + #[serde(default)]
84 + pub cache_salt: Option<String>,
85 + #[serde(default)]
86 + pub data_parallel_rank: Option<u32>,
87 + /// Unsupported in the first-stage Rust client because Python uses a custom
88 + /// tensor/aux-frame encoding path for this field.
89 + #[serde(default)]
90 + pub prompt_embeds: Option<OpaqueValue>,
91 + /// Per-position mask for mixed-mode inputs (e.g. chat completion with
92 + /// `prompt_embeds` content parts). `Some(true)` means real token id;
93 + /// `Some(false)` means the position uses a pre-computed entry from
94 + /// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
95 + #[serde(default)]
96 + pub prompt_is_token_ids: Option<Vec<bool>>,
97 + /// Index of the client, used to ensure outputs are sent back to the same
98 + /// client when scaling out the frontend.
99 + #[serde(default)]
100 + pub client_index: u32,
101 + /// In DP mode, indicates which wave this request is expected to belong to.
102 + #[serde(default)]
103 + pub current_wave: u32,
104 + #[serde(default)]
105 + pub priority: i32,
106 + #[serde(default)]
107 + pub trace_headers: Option<BTreeMap<String, String>>,
108 + #[serde(default)]
109 + pub resumable: bool,
110 + /// Original user-provided request ID, used for output reporting and aborts.
111 + #[serde(default)]
112 + pub external_req_id: Option<String>,
113 + #[serde(default)]
114 + pub reasoning_ended: Option<bool>,
115 + /// Reasoning-parser kwargs forwarded from the frontend to the
116 + /// structured-output backend.
117 + #[serde(default)]
118 + pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
119 + /// If `true`, the request should be added to the scheduler's waiting queue
120 + /// and immediately aborted, so connector-side cleanup runs via the
121 + /// standard `request_finished` hook.
122 + #[serde(default)]
123 + pub abort_immediately: bool,
124 +}
125 +
126 +impl EngineCoreRequest {
127 + /// Validate fields intentionally not supported in the first-stage client.
128 + pub fn validate(&self) -> Result<()> {
129 + if self.prompt_embeds.is_some() {
130 + return Err(Error::UnsupportedField {
131 + context: "EngineCoreRequest",
132 + field: "prompt_embeds",
133 + });
134 + }
135 + Ok(())
136 + }
137 +}
138 +
139 +#[cfg(test)]
140 +mod tests {
141 + use rmpv::Value;
142 +
143 + use super::*;
144 + use crate::protocol::sampling::EngineCoreSamplingParams;
145 + use crate::protocol::{decode_value, encode_msgpack};
146 +
147 + #[test]
148 + fn engine_core_request_serializes_as_full_array() {
149 + let request = EngineCoreRequest {
150 + request_id: "req-1".to_string(),
151 + prompt_token_ids: Some(vec![1, 2, 3]),
152 + sampling_params: Some(EngineCoreSamplingParams {
153 + max_tokens: 8,
154 + ..EngineCoreSamplingParams::for_test()
155 + }),
156 + arrival_time: 1234.5,
157 + client_index: 7,
158 + ..EngineCoreRequest::default()
159 + };
160 +
161 + let encoded = encode_msgpack(&request).unwrap();
162 + let value = decode_value(&encoded).unwrap();
163 + let array = match value {
164 + Value::Array(array) => array,
165 + other => panic!("expected array, got {other:?}"),
166 + };
167 +
168 + assert_eq!(array.len(), 20);
169 + assert_eq!(array[0], Value::from("req-1"));
170 + assert_eq!(array[2], Value::Nil);
171 + assert_eq!(array[4], Value::Nil);
172 + assert_eq!(array[10], Value::Nil);
173 + assert_eq!(array[11], Value::from(7));
174 + }
175 +}
rust/src/engine-core-client/src/protocol/sampling.rs new
+211
@@ -0,0 +1,211 @@
1 +use std::collections::{BTreeSet, HashMap};
2 +
3 +use serde::{Deserialize, Serialize};
4 +use serde_default::DefaultFromSerde;
5 +
6 +use crate::protocol::structured_outputs::StructuredOutputsParams;
7 +
8 +fn default_top_p() -> f32 {
9 + 1.0
10 +}
11 +
12 +fn default_repetition_penalty() -> f32 {
13 + 1.0
14 +}
15 +
16 +fn default_temperature() -> f32 {
17 + 1.0
18 +}
19 +
20 +fn default_max_tokens() -> u32 {
21 + 16
22 +}
23 +
24 +/// Engine-core-facing sampling parameters for text generation.
25 +///
26 +/// This is the normalized southbound subset used by the Rust frontend when it
27 +/// talks to Python engine-core over the wire. User-facing request semantics
28 +/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
29 +/// intentionally handled by higher layers before values reach this DTO.
30 +///
31 +/// Original Python definition:
32 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
33 +// Python's SamplingParams is `omit_defaults=True`, so msgpack drops
34 +// default-valued keys; default the whole struct. Per-field fns cover the
35 +// non-zero defaults.
36 +#[serde_with::skip_serializing_none]
37 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, DefaultFromSerde)]
38 +#[serde(default)]
39 +pub struct EngineCoreSamplingParams {
40 + /// Controls randomness. Lower values are more deterministic; zero means
41 + /// greedy sampling.
42 + #[serde(default = "default_temperature")]
43 + pub temperature: f32,
44 + /// Cumulative probability threshold for nucleus sampling.
45 + #[serde(default = "default_top_p")]
46 + pub top_p: f32,
47 + /// Maximum number of top tokens to consider. `0` means all tokens.
48 + pub top_k: u32,
49 + /// Random seed used by the sampler when present.
50 + pub seed: Option<i64>,
51 + /// Maximum number of tokens to generate per output sequence.
52 + #[serde(default = "default_max_tokens")]
53 + pub max_tokens: u32,
54 + /// Minimum number of tokens to generate before EOS or stop-token handling.
55 + pub min_tokens: u32,
56 + /// Maximum number of reasoning ("thinking") tokens to emit before the
57 + /// reasoning section is force-closed. `None` means unlimited; the
58 + /// user-facing `-1` sentinel is normalized to `None` by the frontend before
59 + /// reaching this DTO, so only non-negative values are sent. Enforced
60 + /// engine-side (and only when a reasoning parser is configured).
61 + pub thinking_token_budget: Option<u64>,
62 + /// Number of log probabilities to return per generated token.
63 + ///
64 + /// `None` disables sample logprobs. `-1` requests the full vocabulary.
65 + pub logprobs: Option<i32>,
66 + /// Number of log probabilities to return per prompt token.
67 + ///
68 + /// `None` disables prompt logprobs. `-1` requests the full vocabulary.
69 + pub prompt_logprobs: Option<i32>,
70 + /// Minimum probability threshold for token sampling.
71 + pub min_p: f32,
72 + /// Frequency penalty applied by the sampler.
73 + pub frequency_penalty: f32,
74 + /// Presence penalty applied by the sampler.
75 + pub presence_penalty: f32,
76 + /// Repetition penalty applied by the sampler.
77 + #[serde(default = "default_repetition_penalty")]
78 + pub repetition_penalty: f32,
79 + /// Token IDs that stop generation.
80 + pub stop_token_ids: Vec<u32>,
81 + /// Primary EOS token ID used by engine-core's dedicated EOS stop path.
82 + ///
83 + /// This mirrors Python's internal `_eos_token_id` field and is derived by
84 + /// the frontend from tokenizer/model metadata rather than supplied directly
85 + /// by end users.
86 + #[serde(rename = "_eos_token_id")]
87 + pub eos_token_id: Option<u32>,
88 + /// Complete stop-token set used by engine-core for `min_tokens` masking.
89 + ///
90 + /// This mirrors Python's internal `_all_stop_token_ids` field and should
91 + /// contain explicit `stop_token_ids` plus any frontend-derived EOS token
92 + /// IDs.
93 + #[serde(rename = "_all_stop_token_ids")]
94 + pub all_stop_token_ids: BTreeSet<u32>,
95 + /// Logit biases to apply during sampling.
96 + /// Keys are token IDs
97 + pub logit_bias: Option<HashMap<u32, f32>>,
98 + /// Restrict output to these token IDs only.
99 + pub allowed_token_ids: Option<Vec<u32>>,
100 + /// Tokenized bad words to avoid during generation.
101 + #[serde(rename = "_bad_words_token_ids")]
102 + pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
103 + /// Parameters for configuring structured outputs (guided decoding).
104 + pub structured_outputs: Option<StructuredOutputsParams>,
105 + /// Specific token IDs for which log probabilities should be returned at
106 + /// each position.
107 + ///
108 + /// When set, the engine returns logprobs for exactly these tokens in
109 + /// addition to the sampled/scored token. Mutually exclusive with the
110 + /// `logprobs` count field in practice.
111 + pub logprob_token_ids: Option<Vec<u32>>,
112 + /// If `Some(true)`, the request will not attempt to read from the prefix
113 + /// cache; newly computed blocks may still populate the cache. `None`
114 + /// defers to engine-core defaults.
115 + pub skip_reading_prefix_cache: Option<bool>,
116 + /// Additional request parameters for custom extensions (from `vllm_xargs`).
117 + pub extra_args: Option<HashMap<String, serde_json::Value>>,
118 +}
119 +
120 +impl EngineCoreSamplingParams {
121 + /// Constructs a default sampling params for testing purposes only.
122 + pub fn for_test() -> Self {
123 + Self {
124 + temperature: 1.0,
125 + top_p: 1.0,
126 + top_k: 0,
127 + seed: None,
128 + max_tokens: 65536,
129 + min_tokens: 0,
130 + thinking_token_budget: None,
131 + logprobs: None,
132 + prompt_logprobs: None,
133 + min_p: 0.0,
134 + frequency_penalty: 0.0,
135 + presence_penalty: 0.0,
136 + repetition_penalty: 1.0,
137 + stop_token_ids: Vec::new(),
138 + eos_token_id: None,
139 + all_stop_token_ids: BTreeSet::new(),
140 + logit_bias: None,
141 + allowed_token_ids: None,
142 + bad_words_token_ids: None,
143 + structured_outputs: None,
144 + logprob_token_ids: None,
145 + skip_reading_prefix_cache: None,
146 + extra_args: None,
147 + }
148 + }
149 +}
150 +
151 +#[cfg(test)]
152 +mod tests {
153 + use rmpv::Value;
154 +
155 + use crate::protocol::decode_msgpack;
156 + use crate::protocol::request::EngineCoreRequest;
157 +
158 + /// A real `sampling_params` is a sparse `omit_defaults` map; absent fields
159 + /// must fall back to defaults. `python_compat` can't catch this since Rust
160 + /// encodes full maps (see `engine_core_request_serializes_as_full_array`).
161 + #[test]
162 + fn decodes_sampling_params_with_omitted_defaults() {
163 + let sampling_params = Value::Map(vec![
164 + (
165 + Value::from("stop_token_ids"),
166 + Value::Array(vec![Value::from(151643u32)]),
167 + ),
168 + (Value::from("skip_reading_prefix_cache"), Value::from(false)),
169 + ]);
170 + let request = Value::Array(vec![
171 + Value::from("req-omit-defaults"),
172 + Value::Array(vec![
173 + Value::from(1u32),
174 + Value::from(2u32),
175 + Value::from(3u32),
176 + ]),
177 + Value::Nil,
178 + sampling_params,
179 + Value::Nil,
180 + Value::from(1.0f64),
181 + ]);
182 +
183 + let mut bytes = Vec::new();
184 + rmpv::encode::write_value(&mut bytes, &request).unwrap();
185 +
186 + let decoded: EngineCoreRequest = decode_msgpack(&bytes)
187 + .expect("a real omit_defaults request must decode (regression: missing field)");
188 +
189 + assert_eq!(decoded.request_id, "req-omit-defaults");
190 + let sampling = decoded.sampling_params.expect("sampling params present");
191 +
192 + assert_eq!(sampling.stop_token_ids, vec![151643]);
193 + assert_eq!(sampling.skip_reading_prefix_cache, Some(false));
194 +
195 + // Omitted fields -> Python defaults.
196 + assert_eq!(sampling.temperature, 1.0);
197 + assert_eq!(sampling.top_p, 1.0);
198 + assert_eq!(sampling.top_k, 0);
199 + assert_eq!(sampling.seed, None);
200 + assert_eq!(sampling.max_tokens, 16);
201 + assert_eq!(sampling.min_tokens, 0);
202 + assert_eq!(sampling.min_p, 0.0);
203 + assert_eq!(sampling.frequency_penalty, 0.0);
204 + assert_eq!(sampling.presence_penalty, 0.0);
205 + assert_eq!(sampling.repetition_penalty, 1.0);
206 + assert_eq!(sampling.logprobs, None);
207 + assert_eq!(sampling.prompt_logprobs, None);
208 + assert_eq!(sampling.eos_token_id, None);
209 + assert!(sampling.all_stop_token_ids.is_empty());
210 + }
211 +}
rust/src/engine-core-client/src/protocol/structured_outputs.rs new
+81
@@ -0,0 +1,81 @@
1 +use serde::{Deserialize, Serialize};
2 +
3 +/// Structured-output backend selected for EngineCore grammar compilation.
4 +///
5 +/// Python vLLM stores this in `StructuredOutputsParams._backend` after request
6 +/// validation. The Rust frontend currently always lowers structured-output
7 +/// requests to guidance, while ignoring any user-supplied `_backend` value.
8 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
9 +#[serde(rename_all = "kebab-case")]
10 +pub enum StructuredOutputBackend {
11 + Xgrammar,
12 + #[default]
13 + Guidance,
14 + Outlines,
15 + LmFormatEnforcer,
16 +}
17 +
18 +/// Parameters for configuring structured outputs (guided decoding).
19 +///
20 +/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
21 +/// `json_object`, or `structural_tag`) should be set. The engine-core
22 +/// backend selects the appropriate grammar compiler based on which field
23 +/// is present.
24 +///
25 +/// Original Python definition:
26 +/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
27 +#[serde_with::skip_serializing_none]
28 +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
29 +#[serde(default)]
30 +pub struct StructuredOutputsParams {
31 + /// JSON schema (as a dict/object or JSON string) constraining the output.
32 + pub json: Option<serde_json::Value>,
33 + /// Regular expression the output must match.
34 + pub regex: Option<String>,
35 + /// List of allowed output strings (the model must produce one of these).
36 + pub choice: Option<Vec<String>>,
37 + /// Context-free grammar (in EBNF-like notation) the output must conform to.
38 + pub grammar: Option<String>,
39 + /// When `true`, output must be valid JSON (free-form, no schema).
40 + pub json_object: Option<bool>,
41 + /// Disable any additional whitespace in guided JSON output.
42 + #[serde(skip_serializing_if = "crate::protocol::is_false")]
43 + pub disable_any_whitespace: bool,
44 + /// Disable `additionalProperties` in JSON schema output.
45 + #[serde(skip_serializing_if = "crate::protocol::is_false")]
46 + pub disable_additional_properties: bool,
47 + /// Custom whitespace pattern for guided JSON output.
48 + pub whitespace_pattern: Option<String>,
49 + /// Structural tag configuration (JSON-encoded string).
50 + pub structural_tag: Option<String>,
51 + /// Structured-output backend, mirroring Python's internal `_backend`.
52 + ///
53 + /// User-supplied values are ignored during deserialization. This matches
54 + /// Python's request boundary, where `_backend` is set by validation rather
55 + /// than accepted as a request-level backend selector.
56 + #[serde(
57 + default,
58 + rename = "_backend",
59 + deserialize_with = "serde_with::rust::deserialize_ignore_any"
60 + )]
61 + pub backend: StructuredOutputBackend,
62 +}
63 +
64 +#[cfg(test)]
65 +mod tests {
66 + use super::*;
67 +
68 + #[test]
69 + fn structured_outputs_backend_ignores_deserialized_value() {
70 + let params: StructuredOutputsParams = serde_json::from_value(serde_json::json!({
71 + "json_object": true,
72 + "_backend": "xgrammar",
73 + }))
74 + .unwrap();
75 +
76 + assert_eq!(params.backend, StructuredOutputBackend::Guidance);
77 +
78 + let value = serde_json::to_value(params).unwrap();
79 + assert_eq!(value["_backend"], "guidance");
80 + }
81 +}
rust/src/engine-core-client/src/tests/client.rs
+5 -4
@@ -22,13 +22,14 @@ use crate::protocol::multimodal::{
22 MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
23 SliceSpec,
24 };
25 +use crate::protocol::output::{
26 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, decode_engine_core_outputs,
27 +};
28 +use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
29 +use crate::protocol::sampling::EngineCoreSamplingParams;
30 use crate::protocol::stats::SchedulerStats;
31 use crate::protocol::tensor::WireTensor;
32 use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
28 -use crate::protocol::{
29 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
30 - EngineCoreRequestType, EngineCoreSamplingParams, decode_engine_core_outputs,
31 -};
33 use crate::test_utils::{
34 IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_sockets,
35 setup_mock_engine_with_init, spawn_mock_engine_task,
rust/src/engine-core-client/src/transport.rs
+2 -3
@@ -18,9 +18,8 @@ use crate::error::{Error, Result, bail_unexpected_handshake_message};
18 use crate::protocol::handshake::{
19 EngineCoreReadyResponse, HandshakeAddresses, HandshakeInitMessage, ReadyMessage,
20 };
21 -use crate::protocol::{
22 - EngineCoreOutputs, decode_engine_core_outputs, decode_msgpack, encode_msgpack,
23 -};
21 +use crate::protocol::output::{EngineCoreOutputs, decode_engine_core_outputs};
22 +use crate::protocol::{decode_msgpack, encode_msgpack};
23
24 /// Dedicated single-frame sentinel emitted by Python `EngineCoreProc` when the
25 /// engine dies.
rust/src/llm/examples/external_engine_smoke.rs
+1 -1
@@ -5,7 +5,7 @@ use clap::Parser;
5 use futures::StreamExt as _;
6 use tokio::time::timeout;
7 use tracing_subscriber::EnvFilter;
8 -use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
8 +use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
9 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
10 use vllm_llm::{FinishReason, GenerateOutputStream, GenerateRequest, Llm};
11
rust/src/llm/src/output.rs
+1 -1
@@ -8,7 +8,7 @@ use futures::stream::FusedStream;
8 use futures::{Stream, StreamExt as _, pin_mut};
9 use serde::{Deserialize, Serialize};
10 use vllm_engine_core_client::protocol::logprobs::Logprobs;
11 -use vllm_engine_core_client::protocol::{EngineCoreFinishReason, StopReason};
11 +use vllm_engine_core_client::protocol::output::{EngineCoreFinishReason, StopReason};
12 use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
13
14 use crate::error::Result;
rust/src/llm/src/request.rs
+4 -4
@@ -4,9 +4,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
4 use uuid::Uuid;
5 use vllm_engine_core_client::protocol::lora::LoraRequest;
6 use vllm_engine_core_client::protocol::multimodal::MmFeatures;
7 -use vllm_engine_core_client::protocol::{
8 - EngineCoreRequest, EngineCoreSamplingParams, ReasoningParserKwargs,
9 -};
7 +use vllm_engine_core_client::protocol::request::{EngineCoreRequest, ReasoningParserKwargs};
8 +use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
9
10 use crate::error::{Error, Result};
11
@@ -134,7 +133,8 @@ fn current_unix_timestamp_secs() -> f64 {
133 mod tests {
134 use std::collections::BTreeMap;
135
137 - use vllm_engine_core_client::protocol::{EngineCoreSamplingParams, ReasoningParserKwargs};
136 + use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
137 + use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
138
139 use super::GenerateRequest;
140 use crate::error::Error;
rust/src/llm/src/request_metrics.rs
+6 -4
@@ -1,7 +1,9 @@
1 use std::time::{SystemTime, UNIX_EPOCH};
2
3 +use vllm_engine_core_client::protocol::output::{
4 + EngineCoreEvent, EngineCoreEventType, EngineCoreOutput,
5 +};
6 use vllm_engine_core_client::protocol::stats::PrefillStats;
4 -use vllm_engine_core_client::protocol::{EngineCoreEvent, EngineCoreEventType, EngineCoreOutput};
7 use vllm_metrics::{
8 EngineLabels, FinishedReasonLabels, METRICS, PromptTokenSourceLabels, RequestMetrics,
9 };
@@ -328,8 +330,8 @@ pub(crate) fn current_unix_timestamp_secs() -> f64 {
330
331 #[cfg(test)]
332 mod tests {
333 + use vllm_engine_core_client::protocol::output::{EngineCoreEvent, EngineCoreEventType};
334 use vllm_engine_core_client::protocol::stats::PrefillStats;
332 - use vllm_engine_core_client::protocol::{EngineCoreEvent, EngineCoreEventType};
335
336 use super::{RequestMetricsTracker, diff_or_zero};
337
@@ -341,7 +343,7 @@ mod tests {
343 2,
344 10.0,
345 100.2,
344 - &vllm_engine_core_client::protocol::EngineCoreOutput {
346 + &vllm_engine_core_client::protocol::output::EngineCoreOutput {
347 request_id: "req-1".to_string(),
348 new_token_ids: vec![1],
349 finish_reason: None,
@@ -369,7 +371,7 @@ mod tests {
371 2,
372 11.5,
373 100.4,
372 - &vllm_engine_core_client::protocol::EngineCoreOutput {
374 + &vllm_engine_core_client::protocol::output::EngineCoreOutput {
375 request_id: "req-1".to_string(),
376 new_token_ids: vec![2, 3],
377 finish_reason: None,
rust/src/llm/tests/generate.rs
+5 -3
@@ -9,11 +9,13 @@ use uuid::Uuid;
9 use vllm_engine_core_client::protocol::logprobs::{
10 Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
11 };
12 -use vllm_engine_core_client::protocol::stats::PrefillStats;
13 -use vllm_engine_core_client::protocol::{
12 +use vllm_engine_core_client::protocol::output::{
13 EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
15 - EngineCoreOutputs, EngineCoreRequest, EngineCoreSamplingParams,
14 + EngineCoreOutputs,
15 };
16 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
17 +use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
18 +use vllm_engine_core_client::protocol::stats::PrefillStats;
19 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
20 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
21 use vllm_llm::{
rust/src/mock-engine/src/engine.rs
+4 -3
@@ -11,12 +11,13 @@ use tokio::sync::mpsc;
11 use tokio::task::yield_now;
12 use tokio_util::sync::CancellationToken;
13 use tracing::{debug, info, warn};
14 +use vllm_engine_core_client::protocol::output::{
15 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
16 +};
17 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
18 use vllm_engine_core_client::protocol::utility::{
19 EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope,
20 };
17 -use vllm_engine_core_client::protocol::{
18 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
19 -};
21
22 use super::Opt;
23
rust/src/mock-engine/src/io.rs
+2 -3
@@ -4,10 +4,9 @@ use tokio::sync::mpsc;
4 use tokio_util::sync::CancellationToken;
5 use tracing::warn;
6 use vllm_engine_core_client::mock_engine::MockEngineDataSockets;
7 +use vllm_engine_core_client::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
8 use vllm_engine_core_client::protocol::utility::EngineCoreUtilityRequest;
8 -use vllm_engine_core_client::protocol::{
9 - EngineCoreRequest, EngineCoreRequestType, decode_msgpack, encode_msgpack,
10 -};
9 +use vllm_engine_core_client::protocol::{decode_msgpack, encode_msgpack};
10 use zeromq::{DealerSocket, PushSocket, SocketRecv as _, SocketSend as _, ZmqMessage};
11
12 use crate::engine::{EngineInput, EngineOutput};
rust/src/mock-engine/src/tests.rs
+3 -3
@@ -5,9 +5,9 @@ use anyhow::Result;
5 use futures::StreamExt as _;
6 use tokio::time::timeout;
7 use tokio_util::sync::CancellationToken;
8 -use vllm_engine_core_client::protocol::{
9 - EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
10 -};
8 +use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
9 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
10 +use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
11 use vllm_engine_core_client::test_utils::IpcNamespace;
12 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
13
rust/src/parser/benches/utils/mod.rs
+1 -2
@@ -5,14 +5,13 @@
5
6 mod adapter;
7
8 +pub(super) use adapter::UnifiedToolParserAdapter;
9 use futures::FutureExt as _;
10 use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool};
11 use tool_parser::traits::ToolParser as ExternalToolParser;
12 use vllm_parser::tool::test_utils::collect_stream;
13 use vllm_parser::tool::{Tool, ToolParser};
14
14 -pub(super) use adapter::UnifiedToolParserAdapter;
15 -
15 pub(super) fn openai_tools(tools: &[Tool]) -> Vec<OpenAiTool> {
16 tools
17 .iter()
rust/src/parser/python/src/lib.rs
+2 -1
@@ -231,9 +231,10 @@ fn _rust_tool_parser(m: &Bound<'_, PyModule>) -> PyResult<()> {
231
232 #[cfg(test)]
233 mod tests {
234 - use super::*;
234 use serde_json::json;
235
236 + use super::*;
237 +
238 fn with_python<R>(f: impl for<'py> FnOnce(Python<'py>) -> R) -> R {
239 Python::initialize();
240 Python::attach(f)
rust/src/parser/src/reasoning/seed_oss.rs
+2 -4
@@ -49,10 +49,8 @@ mod tests {
49 use std::sync::Arc;
50
51 use super::SeedOssReasoningParser;
52 - use crate::reasoning::{
53 - ReasoningParser,
54 - tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer},
55 - };
52 + use crate::reasoning::ReasoningParser;
53 + use crate::reasoning::tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer};
54
55 #[test]
56 fn without_prompt_markers_expects_start_token() {
rust/src/parser/src/reasoning/step3p5.rs
+2 -4
@@ -127,10 +127,8 @@ mod tests {
127 use std::sync::Arc;
128
129 use super::Step3p5ReasoningParser;
130 - use crate::reasoning::{
131 - ReasoningParser,
132 - tests::{THINK_START_ID, fake_tokenizer},
133 - };
130 + use crate::reasoning::ReasoningParser;
131 + use crate::reasoning::tests::{THINK_START_ID, fake_tokenizer};
132
133 #[test]
134 fn picks_up_prompt_start_boundary() {
rust/src/parser/src/tool/mod.rs
+2 -2
@@ -14,8 +14,6 @@ mod parameters;
14 mod qwen_coder;
15 #[cfg(any(test, feature = "test-util"))]
16 pub mod test_utils;
17 -use crate::utils;
18 -
17 use std::collections::{BTreeMap, btree_map};
18
19 pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
@@ -35,6 +33,8 @@ use serde::{Deserialize, Serialize};
33 use serde_json::Value;
34 pub use xgrammar_structural_tag::Model as StructuralTagModel;
35
36 +use crate::utils;
37 +
38 /// One function-style tool made available to the model.
39 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
40 pub struct Tool {
rust/src/parser/src/unified/combined.rs
+1 -2
@@ -2,11 +2,10 @@
2
3 use vllm_tokenizer::DynTokenizer;
4
5 +use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
6 use crate::reasoning::ReasoningParser;
7 use crate::tool::{StructuralTagModel, Tool, ToolParser, ToolParserOutput};
8
8 -use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
9 -
9 /// Unified parser that composes existing reasoning and tool parsers.
10 pub struct CombinedParser {
11 reasoning: Option<Box<dyn ReasoningParser>>,
rust/src/parser/src/unified/gemma4.rs
+1 -2
@@ -1,4 +1,5 @@
1 use serde_json::{Map, Number, Value};
2 +use vllm_tokenizer::DynTokenizer;
3 use winnow::ascii::multispace0 as ws0;
4 use winnow::combinator::{alt, delimited, eof, opt, separated, seq, terminated};
5 use winnow::error::{ContextError, ErrMode, ModalResult};
@@ -6,8 +7,6 @@ use winnow::prelude::*;
7 use winnow::stream::{Partial, Stream};
8 use winnow::token::{literal, take_till, take_until};
9
9 -use vllm_tokenizer::DynTokenizer;
10 -
10 use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
11 use crate::reasoning::last_reasoning_boundary;
12 use crate::tool::{Tool, ToolCallDelta};
rust/src/parser/src/unified/mod.rs
+2 -3
@@ -3,13 +3,12 @@
3 mod combined;
4 mod gemma4;
5
6 +pub use combined::CombinedParser;
7 +pub use gemma4::Gemma4UnifiedParser;
8 use thiserror::Error;
9 use thiserror_ext::Macro;
10 use vllm_tokenizer::DynTokenizer;
11
10 -pub use combined::CombinedParser;
11 -pub use gemma4::Gemma4UnifiedParser;
12 -
12 use crate::reasoning::ReasoningError;
13 use crate::tool::{
14 StructuralTagModel, Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput,
rust/src/server/src/error.rs
+1 -2
@@ -1,8 +1,7 @@
1 use axum::Json;
2 use axum::http::StatusCode;
3 use axum::response::{IntoResponse, Response};
4 -use thiserror_ext::AsReport as _;
5 -use thiserror_ext::{Construct, Macro};
4 +use thiserror_ext::{AsReport as _, Construct, Macro};
5
6 use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
7
rust/src/server/src/grpc/convert.rs
+3 -2
@@ -3,7 +3,8 @@
3
4 use tonic::Status;
5 use uuid::Uuid;
6 -use vllm_engine_core_client::protocol::{StopReason, StructuredOutputsParams};
6 +use vllm_engine_core_client::protocol::output::StopReason;
7 +use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
8 use vllm_text::{
9 DecodedLogprobs, DecodedPromptLogprobs, FinishReason, Finished, Prompt, SamplingParams,
10 TextDecodeOptions, TextRequest,
@@ -502,7 +503,7 @@ impl ResponseOpts {
503
504 #[cfg(test)]
505 mod tests {
505 - use vllm_engine_core_client::protocol::StopReason;
506 + use vllm_engine_core_client::protocol::output::StopReason;
507 use vllm_text::{FinishReason, Finished, Prompt};
508
509 use super::pb::finish_info::{FinishReason as PbFinishReason, StopReason as PbStopReason};
rust/src/server/src/grpc/tests.rs
+3 -2
@@ -18,9 +18,10 @@ use vllm_chat::{
18 ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
19 DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
20 };
21 -use vllm_engine_core_client::protocol::{
22 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
21 +use vllm_engine_core_client::protocol::output::{
22 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
23 };
24 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
25 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
26 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
27 use vllm_llm::Llm;
rust/src/server/src/listener.rs
+1 -1
@@ -152,8 +152,8 @@ impl axum::serve::Listener for Listener {
152
153 /// Allow the unified listener to be adaptable to `tls_listener`.
154 impl AsyncAccept for Listener {
155 - type Connection = ListenerIo;
155 type Address = ListenerAddr;
156 + type Connection = ListenerIo;
157 type Error = std::io::Error;
158
159 fn poll_accept(
rust/src/server/src/middleware/offload.rs
+1 -1
@@ -57,9 +57,9 @@ where
57 S::Error: Send + 'static,
58 B: Send + 'static,
59 {
60 - type Response = S::Response;
60 type Error = S::Error;
61 type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
62 + type Response = S::Response;
63
64 fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
65 self.inner.poll_ready(cx)
rust/src/server/src/routes/http_client_tests.rs
+3 -2
@@ -18,9 +18,10 @@ use vllm_chat::{
18 ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
19 DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
20 };
21 -use vllm_engine_core_client::protocol::{
22 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
21 +use vllm_engine_core_client::protocol::output::{
22 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
23 };
24 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
25 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
26 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
27 use vllm_llm::Llm;
rust/src/server/src/routes/openai/chat_completions.rs
+2 -2
@@ -21,7 +21,7 @@ use vllm_chat::{
21 AssistantBlockKind, AssistantMessageExt as _, ChatEvent, ChatEventStream, ChatEventStreamTrait,
22 CollectedAssistantMessage, FinishReason,
23 };
24 -use vllm_engine_core_client::protocol::StopReason;
24 +use vllm_engine_core_client::protocol::output::StopReason;
25
26 use self::convert::{ResponseOptions, prepare_chat_request};
27 use crate::config::ApiServerOptions;
@@ -825,7 +825,7 @@ mod tests {
825 use vllm_chat::{
826 AssistantBlockKind, AssistantContentBlock, AssistantToolCall, ChatEvent, FinishReason,
827 };
828 - use vllm_engine_core_client::protocol::StopReason;
828 + use vllm_engine_core_client::protocol::output::StopReason;
829 use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
830
831 use super::{
rust/src/server/src/routes/openai/utils/structured_outputs.rs
+1 -1
@@ -1,6 +1,6 @@
1 use serde::{Deserialize, Serialize};
2 use serde_json::Value;
3 -use vllm_engine_core_client::protocol::StructuredOutputsParams;
3 +use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
4
5 use crate::error::ApiError;
6
rust/src/server/src/routes/tests.rs
+5 -4
@@ -24,14 +24,15 @@ use vllm_chat::{
24 NewChatOutputProcessorOptions,
25 };
26 use vllm_engine_core_client::mock_engine::default_ready_response;
27 +use vllm_engine_core_client::protocol::decode_value;
28 use vllm_engine_core_client::protocol::logprobs::{
29 Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
30 };
30 -use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
31 -use vllm_engine_core_client::protocol::{
32 - EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason,
33 - decode_value,
31 +use vllm_engine_core_client::protocol::output::{
32 + EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, StopReason,
33 };
34 +use vllm_engine_core_client::protocol::request::EngineCoreRequest;
35 +use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
36 use vllm_engine_core_client::test_utils::{
37 IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready,
38 };
rust/src/server/src/state.rs
+1 -2
@@ -1,6 +1,5 @@
1 -use std::sync::Arc;
2 -use std::sync::OnceLock;
1 use std::sync::atomic::{AtomicU64, Ordering};
2 +use std::sync::{Arc, OnceLock};
3
4 use serde_json::Value;
5 use sha2::{Digest, Sha256};
rust/src/text/src/lower.rs
+3 -3
@@ -3,15 +3,15 @@ use std::collections::BTreeSet;
3 pub(crate) mod logprobs;
4 pub(crate) mod token_ids;
5
6 -use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
6 +use logprobs::validate_logprobs;
7 +use token_ids::{validate_prompt_token_ids, validate_vocab_range};
8 +use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
9 use vllm_llm::GenerateRequest;
10 use vllm_tokenizer::Tokenizer;
11
12 use crate::backend::{SamplingHints, SamplingLimits};
13 use crate::error::{Error, Result};
14 use crate::request::{SamplingParams, TextRequest};
13 -use logprobs::validate_logprobs;
14 -use token_ids::{validate_prompt_token_ids, validate_vocab_range};
15
16 /// One text request after it has been lowered into the raw generate boundary.
17 #[derive(Debug)]
rust/src/text/src/lower/logprobs.rs
+2 -1
@@ -3,9 +3,10 @@
3 //! `-1` is expanded only for bounds checks. The original request values are
4 //! passed through to engine-core.
5
6 -use crate::backend::SamplingLimits;
6 use thiserror::Error;
7
8 +use crate::backend::SamplingLimits;
9 +
10 #[derive(Debug, Error)]
11 pub enum LogprobsError {
12 #[error("{parameter} must be non-negative or -1, got {value}")]
rust/src/text/src/lower/token_ids.rs
+1 -1
@@ -1,7 +1,7 @@
1 use std::result::Result;
2
3 use thiserror::Error;
4 -use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
4 +use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
5
6 use crate::SamplingLimits;
7
rust/src/text/src/output/decoded.rs
+1 -1
@@ -5,7 +5,7 @@ use futures::{Stream, StreamExt};
5 use serde::{Deserialize, Serialize};
6 use tracing::{Level, debug, trace};
7 use vllm_engine_core_client::AbortCause;
8 -use vllm_engine_core_client::protocol::StopReason;
8 +use vllm_engine_core_client::protocol::output::StopReason;
9 use vllm_llm::{FinishReason, GenerateOutput, TokenUsage};
10 use vllm_tokenizer::{DynTokenizer, IncrementalDecoder};
11
rust/src/text/src/request.rs
+2 -1
@@ -5,7 +5,8 @@ use serde::{Deserialize, Serialize};
5 use serde_json::Value;
6 use vllm_engine_core_client::protocol::lora::LoraRequest;
7 use vllm_engine_core_client::protocol::multimodal::MmFeatures;
8 -use vllm_engine_core_client::protocol::{ReasoningParserKwargs, StructuredOutputsParams};
8 +use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
9 +use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
10
11 use crate::error::{Error, Result};
12 use crate::output::TextDecodeOptions;
rust/src/tokenizer/src/hf/added_tokens.rs
+3 -2
@@ -1,11 +1,12 @@
1 +use std::fs;
2 +use std::path::Path;
3 +
4 use serde::{Deserialize, Serialize};
5 use thiserror_ext::AsReport as _;
6 use tracing::warn;
7
8 use crate::Result;
9
7 -use std::{fs, path::Path};
8 -
10 /// Minimal `tokenizer.json` projection used to patch `added_tokens` while
11 /// preserving the rest of the tokenizer definition verbatim.
12 #[derive(Debug, Deserialize, Serialize)]