[Rust Frontend] Introduce unified parser interface & combined parser (#46583)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Jun 25, 2026 at 11:17 UTC
76c3c4ff63d6f8ffb27a4a6895f97bca2c60ec87
74 files changed
+1204
-1745
pyproject.toml
+2
-2
@@ -129,8 +129,8 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer
129
"docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
130
"tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*",
131
"rust/src/chat/src/renderer/deepseek_v32/fixtures/*",
132
- "rust/src/tool-parser/src/gemma4.rs", "rust/src/text/src/output/decoded.rs",
133
- "rust/src/tokenizer/src/incremental.rs", "rust/src/reasoning-parser/src/tests.rs"]
132
+ "rust/src/parser/src/tool/gemma4.rs", "rust/src/text/src/output/decoded.rs",
133
+ "rust/src/tokenizer/src/incremental.rs", "rust/src/parser/src/reasoning/tests.rs"]
134
ignore-hidden = false
135
136
[tool.typos.default]
rust/Cargo.lock
+14
-22
@@ -5733,10 +5733,9 @@ dependencies = [
5733
"uuid",
5734
"vllm-engine-core-client",
5735
"vllm-llm",
5736
- "vllm-reasoning-parser",
5736
+ "vllm-parser",
5737
"vllm-text",
5738
"vllm-tokenizer",
5739
- "vllm-tool-parser",
5739
"xgrammar-structural-tag",
5740
"zeromq",
5741
]
@@ -5872,11 +5871,22 @@ dependencies = [
5871
]
5872
5873
[[package]]
5875
-name = "vllm-reasoning-parser"
5874
+name = "vllm-parser"
5875
version = "0.1.0"
5876
dependencies = [
5877
+ "criterion",
5878
+ "easy-ext",
5879
+ "expect-test",
5880
+ "futures",
5881
+ "openai-protocol",
5882
+ "serde",
5883
+ "serde_json",
5884
"thiserror 2.0.18",
5885
+ "thiserror-ext",
5886
+ "tool-parser",
5887
"vllm-tokenizer",
5888
+ "winnow",
5889
+ "xgrammar-structural-tag",
5890
]
5891
5892
[[package]]
@@ -5978,24 +5988,6 @@ dependencies = [
5988
"tracing",
5989
]
5990
5981
-[[package]]
5982
-name = "vllm-tool-parser"
5983
-version = "0.1.0"
5984
-dependencies = [
5985
- "criterion",
5986
- "easy-ext",
5987
- "expect-test",
5988
- "futures",
5989
- "openai-protocol",
5990
- "serde",
5991
- "serde_json",
5992
- "thiserror 2.0.18",
5993
- "thiserror-ext",
5994
- "tool-parser",
5995
- "winnow",
5996
- "xgrammar-structural-tag",
5997
-]
5998
-
5991
[[package]]
5992
name = "vllm-tool-parser-py"
5993
version = "0.1.0"
@@ -6004,7 +5996,7 @@ dependencies = [
5996
"pythonize",
5997
"serde_json",
5998
"thiserror-ext",
6007
- "vllm-tool-parser",
5999
+ "vllm-parser",
6000
]
6001
6002
[[package]]
rust/Cargo.toml
+3
-5
@@ -7,12 +7,11 @@ members = [
7
"src/managed-engine",
8
"src/metrics",
9
"src/mock-engine",
10
- "src/reasoning-parser",
10
+ "src/parser",
11
+ "src/parser/python",
12
"src/server",
13
"src/text",
14
"src/tokenizer",
14
- "src/tool-parser",
15
- "src/tool-parser/python",
15
]
16
resolver = "3"
17
@@ -117,11 +116,10 @@ vllm-engine-core-client = { path = "src/engine-core-client" }
116
vllm-llm = { path = "src/llm" }
117
vllm-managed-engine = { path = "src/managed-engine" }
118
vllm-metrics = { path = "src/metrics" }
120
-vllm-reasoning-parser = { path = "src/reasoning-parser" }
119
+vllm-parser = { path = "src/parser" }
120
vllm-server = { path = "src/server" }
121
vllm-text = { path = "src/text" }
122
vllm-tokenizer = { path = "src/tokenizer" }
124
-vllm-tool-parser = { path = "src/tool-parser" }
123
winnow = "1.0.2"
124
xgrammar-structural-tag = "0.1.0"
125
zeromq = { version = "0.6.0", default-features = false, features = [
rust/src/chat/Cargo.toml
+1
-2
@@ -30,10 +30,9 @@ trait-set.workspace = true
30
uuid.workspace = true
31
vllm-engine-core-client.workspace = true
32
vllm-llm.workspace = true
33
-vllm-reasoning-parser.workspace = true
33
+vllm-parser.workspace = true
34
vllm-text.workspace = true
35
vllm-tokenizer.workspace = true
36
-vllm-tool-parser.workspace = true
36
xgrammar-structural-tag.workspace = true
37
38
[dev-dependencies]
rust/src/chat/src/output/default/mod.rs
+22
-43
@@ -1,44 +1,34 @@
1
//! Default output processing pipeline.
2
3
-mod reasoning;
3
mod structural_tag;
5
-mod tool;
4
+mod unified;
5
6
use std::sync::Once;
7
9
-use futures::{Stream, StreamExt as _};
8
+use futures::StreamExt as _;
9
use tracing::info;
11
-use trait_set::trait_set;
10
+use vllm_parser::unified::{CombinedParser, UnifiedParser};
11
use vllm_text::tokenizer::DynTokenizer;
12
14
-use self::reasoning::reasoning_event_stream;
13
use self::structural_tag::apply_structural_tag_constraint;
16
-use self::tool::tool_event_stream;
14
+use self::unified::unified_event_stream;
15
use super::structured::structured_chat_event_stream;
16
use crate::error::Result;
19
-use crate::output::{
20
- AssistantEvent, ChatOutputProcessor, ContentEvent, DynChatEventStream,
21
- DynDecodedTextEventStream,
22
-};
17
+use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
18
use crate::parser::ParserSelection;
19
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
20
use crate::parser::tool::{ToolParser, ToolParserFactory};
21
use crate::request::ChatRequest;
22
use crate::{Error, Result as ChatResult};
23
29
-trait_set! {
30
- trait ContentEventStream = Stream<Item = Result<ContentEvent>> + Send + 'static;
31
-}
32
-
24
/// Default request-scoped output processor used by Hugging Face style chat
25
/// backends.
26
///
27
/// This implementation assumes the backend already emitted decoded text deltas,
37
-/// then optionally layers reasoning parsing and tool-call parsing before
28
+/// then optionally layers unified reasoning and tool-call parsing before
29
/// assembling final structured chat events.
30
pub struct DefaultChatOutputProcessor {
40
- reasoning_parser: Option<Box<dyn ReasoningParser>>,
41
- tool_parser: Option<Box<dyn ToolParser>>,
31
+ parser: Box<dyn UnifiedParser>,
32
parallel_tool_calls: bool,
33
}
34
@@ -66,16 +56,17 @@ impl DefaultChatOutputProcessor {
56
} else {
57
None
58
};
69
- let reasoning_parser = Self::resolve_optional_reasoning_parser(
70
- request,
71
- model_id,
72
- tokenizer,
73
- reasoning_parser,
74
- )?;
59
+ let reasoning_parser =
60
+ Self::resolve_optional_reasoning_parser(model_id, tokenizer, reasoning_parser)?;
61
+ let parser: Box<dyn UnifiedParser> =
62
+ Box::new(CombinedParser::new(reasoning_parser, tool_parser));
63
+
64
+ if parser.preserve_special_tokens() {
65
+ request.decode_options.skip_special_tokens = false;
66
+ }
67
68
Ok(Self {
77
- reasoning_parser,
78
- tool_parser,
69
+ parser,
70
parallel_tool_calls: request.parallel_tool_calls,
71
})
72
}
@@ -87,8 +78,7 @@ impl DefaultChatOutputProcessor {
78
/// content is treated as opaque text.
79
pub fn plain_text_only() -> Self {
80
Self {
90
- reasoning_parser: None,
91
- tool_parser: None,
81
+ parser: Box::new(CombinedParser::plain_text_only()),
82
parallel_tool_calls: true,
83
}
84
}
@@ -112,10 +102,6 @@ impl DefaultChatOutputProcessor {
102
103
let parser = factory.create(parser_name, &request.tools)?;
104
115
- if parser.preserve_special_tokens() {
116
- request.decode_options.skip_special_tokens = false;
117
- }
118
-
105
apply_structural_tag_constraint(request, parser.as_ref())?;
106
107
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
@@ -123,7 +109,6 @@ impl DefaultChatOutputProcessor {
109
}
110
111
fn resolve_optional_reasoning_parser(
126
- request: &mut ChatRequest,
112
model_id: &str,
113
tokenizer: DynTokenizer,
114
selection: &ParserSelection,
@@ -142,10 +127,6 @@ impl DefaultChatOutputProcessor {
127
128
let parser = factory.create(parser_name, tokenizer)?;
129
145
- if parser.preserve_special_tokens() {
146
- request.decode_options.skip_special_tokens = false;
147
- }
148
-
130
REASONING_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using reasoning parser"));
131
Ok(Some(parser))
132
}
@@ -156,16 +137,14 @@ static REASONING_PARSER_LOG_ONCE: Once = Once::new();
137
138
impl ChatOutputProcessor for DefaultChatOutputProcessor {
139
/// Transforms a raw generate-output token stream into structured chat
159
- /// events through three sequential stages once text decoding has
140
+ /// events through two sequential stages once text decoding has
141
/// already happened:
142
///
162
- /// 1. [`reasoning_event_stream`] — reasoning/content separation
163
- /// 2. [`tool_event_stream`] — tool-call parsing
164
- /// 3. [`structured_chat_event_stream`] — final block assembly
143
+ /// 1. [`unified_event_stream`] — reasoning and tool-call parsing
144
+ /// 2. [`structured_chat_event_stream`] — final block assembly
145
fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
166
- let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
167
- let tool = tool_event_stream(reasoning, self.tool_parser);
168
- let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);
146
+ let parsed = unified_event_stream(decoded, self.parser);
147
+ let structured = structured_chat_event_stream(parsed, self.parallel_tool_calls);
148
149
Ok(structured.boxed())
150
}
rust/src/chat/src/output/default/reasoning.rs
deleted
-509
@@ -1,509 +0,0 @@
1
-//! Adapts decoded text updates into reasoning-aware assistant deltas.
2
-//!
3
-//! This stage sits between low-level token decoding and final block assembly.
4
-//! It is the only place in the new pipeline that understands reasoning
5
-//! separation: `decoded.rs` still only produces plain text deltas, while later
6
-//! stages consume the semantic `Text` / `Reasoning` split emitted here.
7
-
8
-use asynk_strim_attr::{TryYielder, try_stream};
9
-use futures::{StreamExt as _, pin_mut};
10
-use thiserror_ext::AsReport;
11
-use tracing::warn;
12
-use vllm_text::output::DecodedTextEvent;
13
-
14
-use super::ContentEvent;
15
-use crate::Result;
16
-use crate::error::Error;
17
-use crate::event::AssistantBlockKind;
18
-use crate::output::DecodedTextEventStream;
19
-use crate::parser::reasoning::{ReasoningDelta, ReasoningParser};
20
-
21
-/// Per-stream reasoning parsing state.
22
-struct ReasoningState {
23
- /// Reasoning parser for the current model family.
24
- parser: Box<dyn ReasoningParser>,
25
- /// Whether reasoning parsing has already failed for this stream.
26
- parser_failed: bool,
27
-}
28
-
29
-impl ReasoningState {
30
- /// Create one fresh reasoning-adaptation state for a new streamed response.
31
- fn new(parser: Box<dyn ReasoningParser>) -> Self {
32
- Self {
33
- parser,
34
- parser_failed: false,
35
- }
36
- }
37
-
38
- /// Convert one decoded text delta into zero or more semantic assistant
39
- /// deltas.
40
- fn process_delta(&mut self, delta: String) -> Vec<ContentEvent> {
41
- // If the parser has already failed, skip parsing and return plain text deltas.
42
- if self.parser_failed {
43
- return vec![ContentEvent::TextDelta {
44
- kind: AssistantBlockKind::Text,
45
- delta,
46
- }];
47
- }
48
-
49
- let mut events = Vec::new();
50
-
51
- match self.parser.push(&delta) {
52
- Ok(result) => {
53
- push_reasoning_delta(&mut events, result);
54
- }
55
- Err(error) => {
56
- if !self.parser_failed {
57
- warn!(
58
- error = %error.as_report(),
59
- "reasoning parser failed; falling back to plain text deltas"
60
- );
61
- self.parser_failed = true;
62
- }
63
- push_text_delta(&mut events, AssistantBlockKind::Text, delta);
64
- }
65
- }
66
-
67
- events
68
- }
69
-
70
- /// Initialize parser state once prompt token IDs are available.
71
- fn initialize(&mut self, prompt_token_ids: &[u32]) {
72
- if self.parser_failed {
73
- return;
74
- }
75
-
76
- match self.parser.initialize(prompt_token_ids) {
77
- Ok(()) => {}
78
- Err(error) => {
79
- warn!(
80
- error = %error.as_report(),
81
- "failed to initialize reasoning parser; falling back to plain text deltas"
82
- );
83
- self.parser_failed = true;
84
- }
85
- }
86
- }
87
-
88
- /// Flush any parser-held partial delimiter state at end of stream.
89
- fn finish(&mut self) -> Vec<ContentEvent> {
90
- if self.parser_failed {
91
- return Vec::new();
92
- }
93
-
94
- match self.parser.finish() {
95
- Ok(result) => {
96
- let mut events = Vec::new();
97
- push_reasoning_delta(&mut events, result);
98
- events
99
- }
100
- Err(error) => {
101
- warn!(error = %error.as_report(), "failed to flush reasoning parser state");
102
- Vec::new()
103
- }
104
- }
105
- }
106
-}
107
-
108
-/// Push one semantic text delta if it is non-empty.
109
-fn push_text_delta(events: &mut Vec<ContentEvent>, kind: AssistantBlockKind, delta: String) {
110
- if delta.is_empty() {
111
- return;
112
- }
113
- events.push(ContentEvent::TextDelta { kind, delta });
114
-}
115
-
116
-/// Convert one parsed reasoning delta into zero or more content events.
117
-fn push_reasoning_delta(events: &mut Vec<ContentEvent>, delta: ReasoningDelta) {
118
- if let Some(reasoning) = delta.reasoning {
119
- push_text_delta(events, AssistantBlockKind::Reasoning, reasoning);
120
- }
121
- if let Some(content) = delta.content {
122
- push_text_delta(events, AssistantBlockKind::Text, content);
123
- }
124
-}
125
-
126
-/// Wrap one decoded-text stream into the internal reasoning event stream.
127
-#[try_stream]
128
-pub(crate) async fn reasoning_event_stream(
129
- decoded_stream: impl DecodedTextEventStream,
130
- reasoning_parser: Option<Box<dyn ReasoningParser>>,
131
- mut y: TryYielder<ContentEvent, Error>,
132
-) -> Result<()> {
133
- pin_mut!(decoded_stream);
134
-
135
- // Without a parser, pass through as plain text deltas.
136
- let Some(reasoning_parser) = reasoning_parser else {
137
- while let Some(event) = decoded_stream.next().await.transpose()? {
138
- for next in ContentEvent::from_decoded_plain_text(event) {
139
- y.yield_ok(next).await;
140
- }
141
- }
142
- return Ok(());
143
- };
144
-
145
- let mut state = ReasoningState::new(reasoning_parser);
146
-
147
- while let Some(event) = decoded_stream.next().await.transpose()? {
148
- match event {
149
- DecodedTextEvent::Start {
150
- prompt_token_ids,
151
- prompt_logprobs,
152
- } => {
153
- state.initialize(&prompt_token_ids);
154
- y.yield_ok(ContentEvent::Start {
155
- prompt_token_ids,
156
- prompt_logprobs,
157
- })
158
- .await;
159
- }
160
- DecodedTextEvent::TextDelta {
161
- delta,
162
- token_ids,
163
- logprobs,
164
- finished,
165
- } => {
166
- for next in state.process_delta(delta) {
167
- y.yield_ok(next).await;
168
- }
169
- if logprobs.is_some() || !token_ids.is_empty() {
170
- y.yield_ok(ContentEvent::LogprobsDelta {
171
- logprobs,
172
- token_ids,
173
- })
174
- .await;
175
- }
176
- if let Some(finished) = finished {
177
- for next in state.finish() {
178
- y.yield_ok(next).await;
179
- }
180
- y.yield_ok(ContentEvent::Done {
181
- usage: finished.usage,
182
- finish_reason: finished.finish_reason,
183
- kv_transfer_params: finished.kv_transfer_params,
184
- })
185
- .await;
186
- }
187
- }
188
- }
189
- }
190
- Ok(())
191
-}
192
-
193
-#[cfg(test)]
194
-mod tests {
195
-
196
- use std::sync::Arc;
197
-
198
- use futures::{StreamExt as _, stream};
199
- use vllm_llm::FinishReason;
200
- use vllm_text::output::{
201
- DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
202
- };
203
- use vllm_tokenizer::{DynTokenizer, Tokenizer};
204
-
205
- use super::super::ContentEvent;
206
- use super::reasoning_event_stream;
207
- use crate::event::AssistantBlockKind;
208
- use crate::parser::reasoning::{
209
- ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, names,
210
- };
211
-
212
- struct FakeTokenizer;
213
-
214
- impl Tokenizer for FakeTokenizer {
215
- fn encode(
216
- &self,
217
- text: &str,
218
- _add_special_tokens: bool,
219
- ) -> vllm_tokenizer::Result<Vec<u32>> {
220
- Ok(text.chars().map(u32::from).collect())
221
- }
222
-
223
- fn decode(
224
- &self,
225
- token_ids: &[u32],
226
- _skip_special_tokens: bool,
227
- ) -> vllm_tokenizer::Result<String> {
228
- Ok(token_ids
229
- .iter()
230
- .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
231
- .collect())
232
- }
233
-
234
- fn token_to_id(&self, token: &str) -> Option<u32> {
235
- match token {
236
- "<think>" => Some(1),
237
- "</think>" => Some(2),
238
- _ => None,
239
- }
240
- }
241
- }
242
-
243
- struct FailingReasoningParser {
244
- fail_next: bool,
245
- }
246
-
247
- impl ReasoningParser for FailingReasoningParser {
248
- fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>, ReasoningError>
249
- where
250
- Self: Sized + 'static,
251
- {
252
- Ok(Box::new(Self { fail_next: true }))
253
- }
254
-
255
- fn push(&mut self, _text: &str) -> Result<ReasoningDelta, ReasoningError> {
256
- if self.fail_next {
257
- self.fail_next = false;
258
- return Err(ReasoningError::MissingToken {
259
- token: "<think>".to_string(),
260
- });
261
- }
262
- Ok(ReasoningDelta::default())
263
- }
264
- }
265
-
266
- fn test_reasoning_parser(factory: &mut ReasoningParserFactory) -> Box<dyn ReasoningParser> {
267
- factory.register_parser::<FailingReasoningParser>("failing");
268
-
269
- factory.create("failing", Arc::new(FakeTokenizer)).unwrap()
270
- }
271
-
272
- #[tokio::test]
273
- async fn reasoning_parser_failure_falls_back_to_plain_text() {
274
- let mut factory = ReasoningParserFactory::new();
275
- let events = stream::iter(vec![
276
- Ok(DecodedTextEvent::Start {
277
- prompt_token_ids: vec![1, 2, 3].into(),
278
- prompt_logprobs: None,
279
- }),
280
- Ok(DecodedTextEvent::TextDelta {
281
- delta: "abc".to_string(),
282
- token_ids: vec![],
283
- logprobs: None,
284
- finished: None,
285
- }),
286
- Ok(DecodedTextEvent::TextDelta {
287
- delta: "def".to_string(),
288
- token_ids: vec![],
289
- logprobs: None,
290
- finished: Some(vllm_text::Finished {
291
- usage: vllm_llm::TokenUsage {
292
- prompt_token_count: 3,
293
- output_token_count: 0,
294
- cached_token_count: 0,
295
- },
296
- finish_reason: FinishReason::stop_eos(),
297
- kv_transfer_params: None,
298
- }),
299
- }),
300
- ]);
301
-
302
- let collected = reasoning_event_stream(events, Some(test_reasoning_parser(&mut factory)))
303
- .collect::<Vec<_>>()
304
- .await;
305
-
306
- let events = collected
307
- .into_iter()
308
- .collect::<crate::Result<Vec<_>>>()
309
- .expect("reasoning stream should not fail");
310
-
311
- assert_eq!(
312
- events,
313
- vec![
314
- ContentEvent::Start {
315
- prompt_token_ids: vec![1, 2, 3].into(),
316
- prompt_logprobs: None,
317
- },
318
- ContentEvent::TextDelta {
319
- kind: AssistantBlockKind::Text,
320
- delta: "abc".to_string(),
321
- },
322
- ContentEvent::TextDelta {
323
- kind: AssistantBlockKind::Text,
324
- delta: "def".to_string(),
325
- },
326
- ContentEvent::Done {
327
- usage: vllm_llm::TokenUsage {
328
- prompt_token_count: 3,
329
- output_token_count: 0,
330
- cached_token_count: 0,
331
- },
332
- finish_reason: FinishReason::stop_eos(),
333
- kv_transfer_params: None,
334
- },
335
- ]
336
- );
337
- }
338
-
339
- #[tokio::test]
340
- async fn reasoning_stream_preserves_logprobs_delta() {
341
- let events = stream::iter(vec![
342
- Ok(DecodedTextEvent::Start {
343
- prompt_token_ids: vec![1].into(),
344
- prompt_logprobs: None,
345
- }),
346
- Ok(DecodedTextEvent::TextDelta {
347
- delta: "abc".to_string(),
348
- token_ids: vec![],
349
- logprobs: Some(DecodedLogprobs {
350
- positions: vec![DecodedPositionLogprobs {
351
- entries: vec![DecodedTokenLogprob {
352
- token_id: 0,
353
- token: "a".to_string(),
354
- logprob: -0.1,
355
- rank: 1,
356
- }],
357
- }],
358
- }),
359
- finished: None,
360
- }),
361
- ]);
362
-
363
- let collected = reasoning_event_stream(events, None)
364
- .collect::<Vec<_>>()
365
- .await
366
- .into_iter()
367
- .collect::<crate::Result<Vec<_>>>()
368
- .unwrap();
369
-
370
- assert_eq!(
371
- collected,
372
- vec![
373
- ContentEvent::Start {
374
- prompt_token_ids: vec![1].into(),
375
- prompt_logprobs: None,
376
- },
377
- ContentEvent::TextDelta {
378
- kind: AssistantBlockKind::Text,
379
- delta: "abc".to_string(),
380
- },
381
- ContentEvent::LogprobsDelta {
382
- logprobs: Some(DecodedLogprobs {
383
- positions: vec![DecodedPositionLogprobs {
384
- entries: vec![DecodedTokenLogprob {
385
- token_id: 0,
386
- token: "a".to_string(),
387
- logprob: -0.1,
388
- rank: 1,
389
- }],
390
- }],
391
- }),
392
- token_ids: vec![],
393
- },
394
- ]
395
- );
396
- }
397
-
398
- #[tokio::test]
399
- async fn qwen3_parser_uses_prompt_end_marker_to_switch_to_content() {
400
- let tokenizer = Arc::new(FakeTokenizer);
401
- let events = stream::iter(vec![
402
- Ok(DecodedTextEvent::Start {
403
- prompt_token_ids: vec![2].into(),
404
- prompt_logprobs: None,
405
- }),
406
- Ok(DecodedTextEvent::TextDelta {
407
- delta: "thought ".to_string(),
408
- token_ids: vec![],
409
- logprobs: None,
410
- finished: None,
411
- }),
412
- Ok(DecodedTextEvent::TextDelta {
413
- delta: "done</think>OK".to_string(),
414
- token_ids: vec![],
415
- logprobs: None,
416
- finished: None,
417
- }),
418
- ]);
419
-
420
- let factory = ReasoningParserFactory::new();
421
- let collected = reasoning_event_stream(
422
- events,
423
- Some(factory.create(names::QWEN3, tokenizer).unwrap()),
424
- )
425
- .collect::<Vec<_>>()
426
- .await;
427
-
428
- let events = collected
429
- .into_iter()
430
- .collect::<crate::Result<Vec<_>>>()
431
- .expect("reasoning stream should not fail");
432
-
433
- assert_eq!(
434
- events,
435
- vec![
436
- ContentEvent::Start {
437
- prompt_token_ids: vec![2].into(),
438
- prompt_logprobs: None,
439
- },
440
- ContentEvent::TextDelta {
441
- kind: AssistantBlockKind::Text,
442
- delta: "thought ".to_string(),
443
- },
444
- ContentEvent::TextDelta {
445
- kind: AssistantBlockKind::Text,
446
- delta: "done</think>OK".to_string(),
447
- },
448
- ]
449
- );
450
- }
451
-
452
- #[tokio::test]
453
- async fn qwen3_parser_tolerates_prompt_prefill_reasoning() {
454
- let tokenizer = Arc::new(FakeTokenizer);
455
- let events = stream::iter(vec![
456
- Ok(DecodedTextEvent::Start {
457
- prompt_token_ids: vec![1].into(),
458
- prompt_logprobs: None,
459
- }),
460
- Ok(DecodedTextEvent::TextDelta {
461
- delta: "thought ".to_string(),
462
- token_ids: vec![],
463
- logprobs: None,
464
- finished: None,
465
- }),
466
- Ok(DecodedTextEvent::TextDelta {
467
- delta: "done</think>OK".to_string(),
468
- token_ids: vec![],
469
- logprobs: None,
470
- finished: None,
471
- }),
472
- ]);
473
-
474
- let factory = ReasoningParserFactory::new();
475
- let collected = reasoning_event_stream(
476
- events,
477
- Some(factory.create(names::QWEN3, tokenizer).unwrap()),
478
- )
479
- .collect::<Vec<_>>()
480
- .await;
481
-
482
- let events = collected
483
- .into_iter()
484
- .collect::<crate::Result<Vec<_>>>()
485
- .expect("reasoning stream should not fail");
486
-
487
- assert_eq!(
488
- events,
489
- vec![
490
- ContentEvent::Start {
491
- prompt_token_ids: vec![1].into(),
492
- prompt_logprobs: None,
493
- },
494
- ContentEvent::TextDelta {
495
- kind: AssistantBlockKind::Reasoning,
496
- delta: "thought ".to_string(),
497
- },
498
- ContentEvent::TextDelta {
499
- kind: AssistantBlockKind::Reasoning,
500
- delta: "done".to_string(),
501
- },
502
- ContentEvent::TextDelta {
503
- kind: AssistantBlockKind::Text,
504
- delta: "OK".to_string(),
505
- },
506
- ]
507
- );
508
- }
509
-}
rust/src/chat/src/output/default/structural_tag.rs
+1
-1
@@ -77,7 +77,7 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagTool
77
mod tests {
78
use serde_json::{Value, json};
79
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
80
- use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
80
+ use vllm_parser::tool::{Qwen3CoderToolParser, Tool};
81
82
use super::*;
83
rust/src/chat/src/output/default/tool.rs
deleted
-985
@@ -1,985 +0,0 @@
1
-//! Adapts plain assistant text deltas into tool-call-aware assistant updates.
2
-//!
3
-//! This stage runs after reasoning separation and before final block assembly.
4
-//! It only inspects normal assistant text, leaves reasoning deltas untouched,
5
-//! and translates incremental tool parsing output into internal tool-call
6
-//! events while preserving plain-text fallback behavior.
7
-
8
-use asynk_strim_attr::{TryYielder, try_stream};
9
-use futures::{StreamExt as _, pin_mut};
10
-use thiserror_ext::AsReport;
11
-use tracing::warn;
12
-
13
-use super::{AssistantEvent, ContentEvent, ContentEventStream};
14
-use crate::Result;
15
-use crate::error::Error;
16
-use crate::event::AssistantBlockKind;
17
-use crate::output::generate_tool_call_id;
18
-use crate::parser::tool::{ToolCallDelta, ToolParser, ToolParserOutput};
19
-
20
-/// Per-stream tool parsing state.
21
-struct ToolState {
22
- /// Parser for the current model family.
23
- parser: Box<dyn ToolParser>,
24
- /// Whether tool parsing has already failed for this stream.
25
- parser_failed: bool,
26
- /// The parser-local index of the currently open tool call, if any.
27
- // NOTE: We only allow single open tool call at a time right now, since that's what all
28
- // supported parsers currently emit. Change this to a `BTreeMap` if we need to support multiple
29
- // interleaved calls in the future.
30
- open_call_index: Option<usize>,
31
-}
32
-
33
-impl ToolState {
34
- /// Create one fresh tool-parsing state for a new streamed response.
35
- fn new(parser: Box<dyn ToolParser>) -> Self {
36
- Self {
37
- parser,
38
- parser_failed: false,
39
- open_call_index: None,
40
- }
41
- }
42
-
43
- /// Convert one semantic assistant text delta into zero or more tool-aware
44
- /// internal events.
45
- fn process_text_delta(
46
- &mut self,
47
- kind: AssistantBlockKind,
48
- delta: String,
49
- ) -> Result<Vec<AssistantEvent>> {
50
- let mut events = Vec::new();
51
-
52
- // Only normal assistant text is eligible for tool parsing. Reasoning
53
- // blocks and plain-text fallback should pass through unchanged.
54
- if kind != AssistantBlockKind::Text || self.parser_failed {
55
- self.open_call_index = None;
56
- events.push(AssistantEvent::TextDelta { kind, delta });
57
- return Ok(events);
58
- }
59
-
60
- let mut output = ToolParserOutput::default();
61
- let parse_result = self.parser.parse_into(&delta, &mut output);
62
-
63
- match parse_result {
64
- Ok(()) => self.process_parser_output(kind, output, &mut events)?,
65
- Err(error) => {
66
- warn!(
67
- error = %error.as_report(),
68
- "tool parser failed; falling back to plain text deltas"
69
- );
70
- // Permanently mark this parser as failed.
71
- // TODO: we may consider recovering from parsing errors in the future.
72
- self.parser_failed = true;
73
-
74
- // On parsing failure, we still apply the partial parser output if any, but we close
75
- // any open tool calls and emit the remaining buffered text as a plain-text delta to
76
- // preserve as much of the output as possible.
77
- self.process_parser_output(kind, output, &mut events)?;
78
- self.open_call_index = None;
79
- push_text_delta(&mut events, kind, self.parser.reset());
80
- }
81
- }
82
-
83
- Ok(events)
84
- }
85
-
86
- /// Apply one parsed tool output to the current stream state.
87
- fn process_parser_output(
88
- &mut self,
89
- kind: AssistantBlockKind,
90
- output: ToolParserOutput,
91
- events: &mut Vec<AssistantEvent>,
92
- ) -> Result<()> {
93
- // When we are not currently streaming a tool call, preserve plain
94
- // text first and then surface any new tool call items.
95
- if self.open_call_index.is_none() {
96
- push_text_delta(events, kind, output.normal_text);
97
- self.process_tool_items(output.calls, events)?;
98
- } else {
99
- // Once a tool call is open, prioritize tool deltas first. If the
100
- // parser emits normal text again, close the tool call and resume
101
- // plain text output.
102
- self.process_tool_items(output.calls, events)?;
103
- if !output.normal_text.is_empty() {
104
- self.open_call_index = None;
105
- push_text_delta(events, kind, output.normal_text);
106
- }
107
- }
108
- Ok(())
109
- }
110
-
111
- /// Apply one batch of parsed tool-call deltas emitted by the parser.
112
- fn process_tool_items(
113
- &mut self,
114
- items: Vec<ToolCallDelta>,
115
- events: &mut Vec<AssistantEvent>,
116
- ) -> Result<()> {
117
- for item in items {
118
- if let Some(name) = item.name {
119
- let is_new_tool = match self.open_call_index {
120
- Some(open_call_index) => open_call_index != item.tool_index,
121
- None => true,
122
- };
123
- if is_new_tool {
124
- let id = self
125
- .parser
126
- .tool_call_id(item.tool_index)
127
- .map(str::to_string)
128
- .unwrap_or_else(generate_tool_call_id);
129
- self.open_call_index = Some(item.tool_index);
130
- events.push(AssistantEvent::ToolCallStart { id, name });
131
- }
132
- }
133
-
134
- if item.arguments.is_empty() {
135
- // No arguments delta to apply.
136
- continue;
137
- }
138
- let Some(open_call_index) = self.open_call_index else {
139
- return Err(Error::ToolCallStreamInvariant {
140
- message: format!(
141
- "received arguments for tool index {} before any tool-call start",
142
- item.tool_index
143
- ),
144
- });
145
- };
146
- if open_call_index != item.tool_index {
147
- return Err(Error::ToolCallStreamInvariant {
148
- message: format!(
149
- "received arguments for tool index {} while tool index {} is open",
150
- item.tool_index, open_call_index
151
- ),
152
- });
153
- }
154
-
155
- events.push(AssistantEvent::ToolCallArgumentsDelta {
156
- delta: item.arguments,
157
- });
158
- }
159
- Ok(())
160
- }
161
-
162
- /// Flush parser state at end-of-stream and close any remaining open calls.
163
- fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
164
- let mut events = Vec::new();
165
-
166
- if self.parser_failed {
167
- return Ok(events);
168
- }
169
-
170
- match self.parser.finish() {
171
- Ok(output) => {
172
- self.process_parser_output(AssistantBlockKind::Text, output, &mut events)?
173
- }
174
- Err(error) => {
175
- warn!(
176
- error = %error.as_report(),
177
- "tool parser finish failed; closing open tool calls with buffered state"
178
- );
179
- self.parser_failed = true;
180
- }
181
- }
182
-
183
- Ok(events)
184
- }
185
-}
186
-
187
-/// Push one plain-text delta if it is non-empty.
188
-fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
189
- if delta.is_empty() {
190
- return;
191
- }
192
- events.push(AssistantEvent::TextDelta { kind, delta });
193
-}
194
-
195
-/// Wrap one semantic assistant stream into the internal tool-aware assistant
196
-/// stream.
197
-#[try_stream]
198
-pub(crate) async fn tool_event_stream(
199
- stream: impl ContentEventStream,
200
- parser: Option<Box<dyn ToolParser>>,
201
- mut y: TryYielder<AssistantEvent, Error>,
202
-) -> Result<()> {
203
- // Without a parser, pass through the input stream unchanged.
204
- let Some(parser) = parser else {
205
- pin_mut!(stream);
206
- while let Some(event) = stream.next().await.transpose()? {
207
- y.yield_ok(event.into()).await;
208
- }
209
- return Ok(());
210
- };
211
-
212
- pin_mut!(stream);
213
- let mut state = ToolState::new(parser);
214
-
215
- while let Some(event) = stream.next().await.transpose()? {
216
- match event {
217
- ContentEvent::Start {
218
- prompt_token_ids,
219
- prompt_logprobs,
220
- } => {
221
- y.yield_ok(AssistantEvent::Start {
222
- prompt_token_ids,
223
- prompt_logprobs,
224
- })
225
- .await;
226
- }
227
- ContentEvent::TextDelta { kind, delta } => {
228
- for next in state.process_text_delta(kind, delta)? {
229
- y.yield_ok(next).await;
230
- }
231
- }
232
- ContentEvent::LogprobsDelta {
233
- logprobs,
234
- token_ids,
235
- } => {
236
- y.yield_ok(AssistantEvent::LogprobsDelta {
237
- logprobs,
238
- token_ids,
239
- })
240
- .await;
241
- }
242
- ContentEvent::Done {
243
- usage,
244
- finish_reason,
245
- kv_transfer_params,
246
- } => {
247
- for next in state.finish()? {
248
- y.yield_ok(next).await;
249
- }
250
-
251
- y.yield_ok(AssistantEvent::Done {
252
- usage,
253
- finish_reason,
254
- kv_transfer_params,
255
- })
256
- .await;
257
- }
258
- }
259
- }
260
- Ok(())
261
-}
262
-
263
-#[cfg(test)]
264
-mod tests {
265
-
266
- use futures::{StreamExt as _, stream};
267
- use vllm_llm::FinishReason;
268
- use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
269
- use vllm_tool_parser::Result;
270
-
271
- use super::super::{AssistantEvent, ContentEvent};
272
- use super::tool_event_stream;
273
- use crate::error::Error;
274
- use crate::event::{AssistantBlockKind, AssistantMessageExt as _};
275
- use crate::output::structured::structured_chat_event_stream;
276
- use crate::parser::tool::{
277
- DeepSeekV4ToolParser, ToolParser, ToolParserError, ToolParserOutput,
278
- };
279
- use crate::request::ChatTool;
280
- use crate::stream::{ChatEventStream, CollectedAssistantMessage};
281
-
282
- struct FailingParser {
283
- fail_next: bool,
284
- buffered: String,
285
- }
286
-
287
- struct ScriptedParser {
288
- push_outputs: Vec<ToolParserOutput>,
289
- finish_output: ToolParserOutput,
290
- }
291
-
292
- struct PartialThenFailParser {
293
- buffered: String,
294
- }
295
-
296
- struct IdScriptedParser {
297
- output: ToolParserOutput,
298
- tool_call_id: Option<String>,
299
- }
300
-
301
- impl ToolParser for FailingParser {
302
- fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
303
- where
304
- Self: Sized + 'static,
305
- {
306
- Ok(Box::new(Self {
307
- fail_next: false,
308
- buffered: String::new(),
309
- }))
310
- }
311
-
312
- fn parse_into(&mut self, chunk: &str, _output: &mut ToolParserOutput) -> Result<()> {
313
- self.buffered.push_str(chunk);
314
- if self.fail_next {
315
- self.fail_next = false;
316
- return Err(ToolParserError::ParsingFailed {
317
- message: "boom".to_string(),
318
- });
319
- }
320
-
321
- self.buffered.clear();
322
- Ok(())
323
- }
324
-
325
- fn finish(&mut self) -> Result<ToolParserOutput> {
326
- Ok(ToolParserOutput::default())
327
- }
328
-
329
- fn reset(&mut self) -> String {
330
- std::mem::take(&mut self.buffered)
331
- }
332
- }
333
-
334
- impl ToolParser for ScriptedParser {
335
- fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
336
- where
337
- Self: Sized + 'static,
338
- {
339
- Ok(Box::new(Self {
340
- push_outputs: Vec::new(),
341
- finish_output: ToolParserOutput::default(),
342
- }))
343
- }
344
-
345
- fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
346
- let mut next = self.push_outputs.pop().unwrap_or_default();
347
- output.normal_text.push_str(&next.normal_text);
348
- output.calls.append(&mut next.calls);
349
- Ok(())
350
- }
351
-
352
- fn finish(&mut self) -> Result<ToolParserOutput> {
353
- Ok(std::mem::take(&mut self.finish_output))
354
- }
355
-
356
- fn reset(&mut self) -> String {
357
- String::new()
358
- }
359
- }
360
-
361
- impl ToolParser for IdScriptedParser {
362
- fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
363
- where
364
- Self: Sized + 'static,
365
- {
366
- Ok(Box::new(Self {
367
- output: ToolParserOutput::default(),
368
- tool_call_id: None,
369
- }))
370
- }
371
-
372
- fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
373
- (tool_index == 0).then_some(self.tool_call_id.as_deref()).flatten()
374
- }
375
-
376
- fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
377
- output.append(std::mem::take(&mut self.output));
378
- Ok(())
379
- }
380
-
381
- fn finish(&mut self) -> Result<ToolParserOutput> {
382
- Ok(ToolParserOutput::default())
383
- }
384
-
385
- fn reset(&mut self) -> String {
386
- String::new()
387
- }
388
- }
389
-
390
- impl ToolParser for PartialThenFailParser {
391
- fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>
392
- where
393
- Self: Sized + 'static,
394
- {
395
- Ok(Box::new(Self {
396
- buffered: String::new(),
397
- }))
398
- }
399
-
400
- fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
401
- output.calls.extend([
402
- crate::parser::tool::ToolCallDelta {
403
- tool_index: 0,
404
- name: Some("get_weather".to_string()),
405
- arguments: String::new(),
406
- },
407
- crate::parser::tool::ToolCallDelta {
408
- tool_index: 0,
409
- name: None,
410
- arguments: r#"{"location":"SF"}"#.to_string(),
411
- },
412
- ]);
413
- self.buffered.push_str(" trailing text");
414
- Err(ToolParserError::ParsingFailed {
415
- message: "boom".to_string(),
416
- })
417
- }
418
-
419
- fn finish(&mut self) -> Result<ToolParserOutput> {
420
- Ok(ToolParserOutput::default())
421
- }
422
-
423
- fn reset(&mut self) -> String {
424
- std::mem::take(&mut self.buffered)
425
- }
426
- }
427
-
428
- fn deepseek_v4_test_tools() -> Vec<ChatTool> {
429
- vec![
430
- ChatTool {
431
- name: "get_weather".to_string(),
432
- description: None,
433
- parameters: serde_json::json!({
434
- "type": "object",
435
- "properties": {
436
- "location": { "type": "string" }
437
- }
438
- }),
439
- strict: None,
440
- },
441
- ChatTool {
442
- name: "add".to_string(),
443
- description: None,
444
- parameters: serde_json::json!({
445
- "type": "object",
446
- "properties": {
447
- "x": { "type": "integer" },
448
- "y": { "type": "integer" }
449
- }
450
- }),
451
- strict: None,
452
- },
453
- ]
454
- }
455
-
456
- async fn collect_deepseek_v4_message(chunks: Vec<String>) -> CollectedAssistantMessage {
457
- let events = chunks
458
- .into_iter()
459
- .map(|delta| {
460
- Ok(ContentEvent::TextDelta {
461
- kind: AssistantBlockKind::Text,
462
- delta,
463
- })
464
- })
465
- .chain(std::iter::once(Ok(ContentEvent::Done {
466
- usage: vllm_llm::TokenUsage {
467
- prompt_token_count: 1,
468
- output_token_count: 1,
469
- cached_token_count: 0,
470
- },
471
- finish_reason: FinishReason::stop_eos(),
472
- kv_transfer_params: None,
473
- })));
474
- let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
475
- let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
476
- let chat_events = structured_chat_event_stream(assistant_events, true);
477
-
478
- ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
479
- .collect_message()
480
- .await
481
- .unwrap()
482
- }
483
-
484
- fn message_tool_projection(
485
- message: &CollectedAssistantMessage,
486
- ) -> (String, Vec<(String, serde_json::Value)>) {
487
- (
488
- message.message.text(),
489
- message
490
- .message
491
- .tool_calls()
492
- .map(|call| {
493
- (
494
- call.name.clone(),
495
- serde_json::from_str(&call.arguments).unwrap(),
496
- )
497
- })
498
- .collect(),
499
- )
500
- }
501
-
502
- #[tokio::test]
503
- async fn tool_parser_error_preserves_partial_output_and_flushes_buffer() {
504
- let events = stream::iter(vec![
505
- Ok(ContentEvent::TextDelta {
506
- kind: AssistantBlockKind::Text,
507
- delta: "ignored".to_string(),
508
- }),
509
- Ok(ContentEvent::Done {
510
- usage: vllm_llm::TokenUsage {
511
- prompt_token_count: 1,
512
- output_token_count: 1,
513
- cached_token_count: 0,
514
- },
515
- finish_reason: FinishReason::stop_eos(),
516
- kv_transfer_params: None,
517
- }),
518
- ]);
519
-
520
- let events = tool_event_stream(
521
- events,
522
- Some(Box::new(PartialThenFailParser {
523
- buffered: String::new(),
524
- })),
525
- )
526
- .collect::<Vec<_>>()
527
- .await
528
- .into_iter()
529
- .collect::<crate::Result<Vec<_>>>()
530
- .unwrap();
531
-
532
- assert!(matches!(
533
- &events[0],
534
- AssistantEvent::ToolCallStart { name, .. } if name == "get_weather"
535
- ));
536
- assert!(matches!(
537
- &events[1],
538
- AssistantEvent::ToolCallArgumentsDelta { delta } if delta == r#"{"location":"SF"}"#
539
- ));
540
- assert_eq!(
541
- events[2],
542
- AssistantEvent::TextDelta {
543
- kind: AssistantBlockKind::Text,
544
- delta: " trailing text".to_string(),
545
- }
546
- );
547
- assert!(matches!(events[3], AssistantEvent::Done { .. }));
548
- }
549
-
550
- #[tokio::test]
551
- async fn tool_stream_preserves_parser_provided_tool_call_id() {
552
- let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
553
- kind: AssistantBlockKind::Text,
554
- delta: "ignored".to_string(),
555
- })]);
556
- let parser = IdScriptedParser {
557
- output: ToolParserOutput {
558
- normal_text: String::new(),
559
- calls: vec![crate::parser::tool::ToolCallDelta {
560
- tool_index: 0,
561
- name: Some("get_weather".to_string()),
562
- arguments: "{}".to_string(),
563
- }],
564
- },
565
- tool_call_id: Some("functions.get_weather:0".to_string()),
566
- };
567
-
568
- let events = tool_event_stream(events, Some(Box::new(parser)))
569
- .collect::<Vec<_>>()
570
- .await
571
- .into_iter()
572
- .collect::<crate::Result<Vec<_>>>()
573
- .unwrap();
574
-
575
- assert!(matches!(
576
- &events[0],
577
- AssistantEvent::ToolCallStart { id, name }
578
- if id == "functions.get_weather:0" && name == "get_weather"
579
- ));
580
- }
581
-
582
- #[tokio::test]
583
- async fn tool_stream_generates_tool_call_id_when_parser_omits_one() {
584
- let events = stream::iter(vec![Ok(ContentEvent::TextDelta {
585
- kind: AssistantBlockKind::Text,
586
- delta: "ignored".to_string(),
587
- })]);
588
- let parser = IdScriptedParser {
589
- output: ToolParserOutput {
590
- normal_text: String::new(),
591
- calls: vec![crate::parser::tool::ToolCallDelta {
592
- tool_index: 0,
593
- name: Some("get_weather".to_string()),
594
- arguments: "{}".to_string(),
595
- }],
596
- },
597
- tool_call_id: None,
598
- };
599
-
600
- let events = tool_event_stream(events, Some(Box::new(parser)))
601
- .collect::<Vec<_>>()
602
- .await
603
- .into_iter()
604
- .collect::<crate::Result<Vec<_>>>()
605
- .unwrap();
606
-
607
- assert!(matches!(
608
- &events[0],
609
- AssistantEvent::ToolCallStart { id, name }
610
- if id.starts_with("call_") && name == "get_weather"
611
- ));
612
- }
613
-
614
- #[tokio::test]
615
- async fn real_buffered_parser_error_matches_streaming_and_non_streaming() {
616
- let prefix = "I will check both.\n";
617
- let first_tool_call = concat!(
618
- "<｜DSML｜tool_calls>\n",
619
- "<｜DSML｜invoke name=\"get_weather\">\n",
620
- "<｜DSML｜parameter name=\"location\" string=\"true\">Tokyo</｜DSML｜parameter>\n",
621
- "</｜DSML｜invoke>",
622
- );
623
- let malformed_second_tool_call = concat!(
624
- "\n<｜DSML｜invoke name=\"add\">\n",
625
- "not a parameter\n",
626
- "</｜DSML｜invoke>\n",
627
- "</｜DSML｜tool_calls>",
628
- );
629
- let streaming_chunks = vec![
630
- prefix.to_string(),
631
- first_tool_call.to_string(),
632
- malformed_second_tool_call.to_string(),
633
- ];
634
- let full_output = streaming_chunks.concat();
635
-
636
- let streaming = collect_deepseek_v4_message(streaming_chunks).await;
637
- let non_streaming = collect_deepseek_v4_message(vec![full_output]).await;
638
-
639
- let expected = (
640
- format!("{prefix}{malformed_second_tool_call}"),
641
- vec![(
642
- "get_weather".to_string(),
643
- serde_json::json!({ "location": "Tokyo" }),
644
- )],
645
- );
646
- assert_eq!(message_tool_projection(&streaming), expected);
647
- assert_eq!(message_tool_projection(&non_streaming), expected);
648
- }
649
-
650
- #[tokio::test]
651
- async fn tool_parser_failure_falls_back_to_plain_text() {
652
- let events = stream::iter(vec![
653
- Ok(ContentEvent::Start {
654
- prompt_token_ids: vec![1, 2, 3].into(),
655
- prompt_logprobs: None,
656
- }),
657
- Ok(ContentEvent::TextDelta {
658
- kind: AssistantBlockKind::Text,
659
- delta: "abc".to_string(),
660
- }),
661
- Ok(ContentEvent::TextDelta {
662
- kind: AssistantBlockKind::Text,
663
- delta: "def".to_string(),
664
- }),
665
- Ok(ContentEvent::Done {
666
- usage: vllm_llm::TokenUsage {
667
- prompt_token_count: 3,
668
- output_token_count: 0,
669
- cached_token_count: 0,
670
- },
671
- finish_reason: FinishReason::stop_eos(),
672
- kv_transfer_params: None,
673
- }),
674
- ]);
675
-
676
- let collected = tool_event_stream(
677
- events,
678
- Some(Box::new(FailingParser {
679
- fail_next: true,
680
- buffered: String::new(),
681
- })),
682
- )
683
- .collect::<Vec<_>>()
684
- .await;
685
-
686
- let events = collected
687
- .into_iter()
688
- .collect::<crate::Result<Vec<_>>>()
689
- .expect("tool stream should not fail");
690
-
691
- assert_eq!(
692
- events,
693
- vec![
694
- AssistantEvent::Start {
695
- prompt_token_ids: vec![1, 2, 3].into(),
696
- prompt_logprobs: None,
697
- },
698
- AssistantEvent::TextDelta {
699
- kind: AssistantBlockKind::Text,
700
- delta: "abc".to_string(),
701
- },
702
- AssistantEvent::TextDelta {
703
- kind: AssistantBlockKind::Text,
704
- delta: "def".to_string(),
705
- },
706
- AssistantEvent::Done {
707
- usage: vllm_llm::TokenUsage {
708
- prompt_token_count: 3,
709
- output_token_count: 0,
710
- cached_token_count: 0,
711
- },
712
- finish_reason: FinishReason::stop_eos(),
713
- kv_transfer_params: None,
714
- },
715
- ]
716
- );
717
-
718
- let message = ChatEventStream::new(
719
- "req_fallback".to_string(),
720
- Box::pin(structured_chat_event_stream(
721
- stream::iter(events.into_iter().map(Ok)),
722
- true,
723
- )),
724
- )
725
- .collect_message()
726
- .await
727
- .expect("collect_message should succeed");
728
- assert_eq!(message.message.text(), "abcdef");
729
- assert!(message.message.tool_calls().next().is_none());
730
- }
731
-
732
- #[tokio::test]
733
- async fn tool_stream_preserves_logprobs_delta() {
734
- let events = stream::iter(vec![
735
- Ok(ContentEvent::Start {
736
- prompt_token_ids: vec![1].into(),
737
- prompt_logprobs: None,
738
- }),
739
- Ok(ContentEvent::LogprobsDelta {
740
- logprobs: Some(DecodedLogprobs {
741
- positions: vec![DecodedPositionLogprobs {
742
- entries: vec![DecodedTokenLogprob {
743
- token_id: 0,
744
- token: "a".to_string(),
745
- logprob: -0.2,
746
- rank: 1,
747
- }],
748
- }],
749
- }),
750
- token_ids: vec![],
751
- }),
752
- Ok(ContentEvent::Done {
753
- usage: vllm_llm::TokenUsage {
754
- prompt_token_count: 1,
755
- output_token_count: 0,
756
- cached_token_count: 0,
757
- },
758
- finish_reason: FinishReason::stop_eos(),
759
- kv_transfer_params: None,
760
- }),
761
- ]);
762
- let events = tool_event_stream(
763
- events,
764
- Some(Box::new(FailingParser {
765
- fail_next: false,
766
- buffered: String::new(),
767
- })),
768
- )
769
- .collect::<Vec<_>>()
770
- .await
771
- .into_iter()
772
- .collect::<crate::Result<Vec<_>>>()
773
- .unwrap();
774
-
775
- assert_eq!(
776
- events,
777
- vec![
778
- AssistantEvent::Start {
779
- prompt_token_ids: vec![1].into(),
780
- prompt_logprobs: None,
781
- },
782
- AssistantEvent::LogprobsDelta {
783
- logprobs: Some(DecodedLogprobs {
784
- positions: vec![DecodedPositionLogprobs {
785
- entries: vec![DecodedTokenLogprob {
786
- token_id: 0,
787
- token: "a".to_string(),
788
- logprob: -0.2,
789
- rank: 1,
790
- }],
791
- }],
792
- }),
793
- token_ids: vec![],
794
- },
795
- AssistantEvent::Done {
796
- usage: vllm_llm::TokenUsage {
797
- prompt_token_count: 1,
798
- output_token_count: 0,
799
- cached_token_count: 0,
800
- },
801
- finish_reason: FinishReason::stop_eos(),
802
- kv_transfer_params: None,
803
- },
804
- ]
805
- );
806
- }
807
-
808
- #[tokio::test]
809
- async fn tool_stream_rejects_interleaved_tool_indices() {
810
- let events = stream::iter(vec![
811
- Ok(ContentEvent::TextDelta {
812
- kind: AssistantBlockKind::Text,
813
- delta: "ignored".to_string(),
814
- }),
815
- Ok(ContentEvent::Done {
816
- usage: vllm_llm::TokenUsage {
817
- prompt_token_count: 1,
818
- output_token_count: 1,
819
- cached_token_count: 0,
820
- },
821
- finish_reason: FinishReason::stop_eos(),
822
- kv_transfer_params: None,
823
- }),
824
- ]);
825
-
826
- let parser = ScriptedParser {
827
- push_outputs: vec![ToolParserOutput {
828
- normal_text: String::new(),
829
- calls: vec![
830
- crate::parser::tool::ToolCallDelta {
831
- tool_index: 0,
832
- name: Some("first".to_string()),
833
- arguments: String::new(),
834
- },
835
- crate::parser::tool::ToolCallDelta {
836
- tool_index: 1,
837
- name: None,
838
- arguments: "{}".to_string(),
839
- },
840
- ],
841
- }],
842
- finish_output: ToolParserOutput::default(),
843
- };
844
-
845
- let err = tool_event_stream(events, Some(Box::new(parser)))
846
- .collect::<Vec<_>>()
847
- .await
848
- .into_iter()
849
- .find_map(|output| output.err())
850
- .expect("expected invariant error");
851
-
852
- assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
853
- }
854
-
855
- #[tokio::test]
856
- async fn tool_stream_resets_open_tool_when_normal_text_interrupts_it() {
857
- let events = stream::iter(vec![
858
- Ok(ContentEvent::TextDelta {
859
- kind: AssistantBlockKind::Text,
860
- delta: "start".to_string(),
861
- }),
862
- Ok(ContentEvent::TextDelta {
863
- kind: AssistantBlockKind::Text,
864
- delta: "text".to_string(),
865
- }),
866
- Ok(ContentEvent::TextDelta {
867
- kind: AssistantBlockKind::Text,
868
- delta: "args".to_string(),
869
- }),
870
- ]);
871
-
872
- let parser = ScriptedParser {
873
- push_outputs: vec![
874
- ToolParserOutput {
875
- normal_text: String::new(),
876
- calls: vec![crate::parser::tool::ToolCallDelta {
877
- tool_index: 0,
878
- name: None,
879
- arguments: "}".to_string(),
880
- }],
881
- },
882
- ToolParserOutput {
883
- normal_text: "plain text".to_string(),
884
- calls: Vec::new(),
885
- },
886
- ToolParserOutput {
887
- normal_text: String::new(),
888
- calls: vec![crate::parser::tool::ToolCallDelta {
889
- tool_index: 0,
890
- name: Some("first".to_string()),
891
- arguments: "{".to_string(),
892
- }],
893
- },
894
- ],
895
- finish_output: ToolParserOutput::default(),
896
- };
897
-
898
- let err = tool_event_stream(events, Some(Box::new(parser)))
899
- .collect::<Vec<_>>()
900
- .await
901
- .into_iter()
902
- .find_map(|output| output.err())
903
- .expect("expected invariant error");
904
-
905
- assert!(matches!(
906
- err,
907
- Error::ToolCallStreamInvariant { message }
908
- if message == "received arguments for tool index 0 before any tool-call start"
909
- ));
910
- }
911
-
912
- #[tokio::test]
913
- async fn tool_stream_emits_start_and_args_for_terminal_text() {
914
- let events = stream::iter(vec![
915
- Ok(ContentEvent::Start {
916
- prompt_token_ids: vec![1].into(),
917
- prompt_logprobs: None,
918
- }),
919
- Ok(ContentEvent::TextDelta {
920
- kind: AssistantBlockKind::Text,
921
- delta: "ignored".to_string(),
922
- }),
923
- Ok(ContentEvent::Done {
924
- usage: vllm_llm::TokenUsage {
925
- prompt_token_count: 1,
926
- output_token_count: 1,
927
- cached_token_count: 0,
928
- },
929
- finish_reason: FinishReason::stop_eos(),
930
- kv_transfer_params: None,
931
- }),
932
- ]);
933
-
934
- let parser = ScriptedParser {
935
- push_outputs: vec![ToolParserOutput {
936
- normal_text: String::new(),
937
- calls: vec![
938
- crate::parser::tool::ToolCallDelta {
939
- tool_index: 0,
940
- name: Some("first".to_string()),
941
- arguments: r#"{"a":1}"#.to_string(),
942
- },
943
- crate::parser::tool::ToolCallDelta {
944
- tool_index: 1,
945
- name: Some("second".to_string()),
946
- arguments: r#"{"b":2}"#.to_string(),
947
- },
948
- ],
949
- }],
950
- finish_output: ToolParserOutput::default(),
951
- };
952
-
953
- let events = tool_event_stream(events, Some(Box::new(parser)))
954
- .collect::<Vec<_>>()
955
- .await
956
- .into_iter()
957
- .collect::<crate::Result<Vec<_>>>()
958
- .unwrap();
959
-
960
- assert!(matches!(events[1], AssistantEvent::ToolCallStart { .. }));
961
- assert!(matches!(
962
- events[2],
963
- AssistantEvent::ToolCallArgumentsDelta { .. }
964
- ));
965
- assert!(matches!(events[3], AssistantEvent::ToolCallStart { .. }));
966
- assert!(matches!(
967
- events[4],
968
- AssistantEvent::ToolCallArgumentsDelta { .. }
969
- ));
970
- let collected = ChatEventStream::new(
971
- "req_final_only".to_string(),
972
- Box::pin(structured_chat_event_stream(
973
- stream::iter(events.into_iter().map(Ok)),
974
- true,
975
- )),
976
- )
977
- .collect_message()
978
- .await
979
- .unwrap();
980
- let tool_calls = collected.message.tool_calls().collect::<Vec<_>>();
981
- assert_eq!(tool_calls.len(), 2);
982
- assert_eq!(tool_calls[0].name, "first");
983
- assert_eq!(tool_calls[1].name, "second");
984
- }
985
-}
rust/src/chat/src/output/default/unified.rs
new
+582
@@ -0,0 +1,582 @@
1
+//! Adapts decoded text updates into parsed assistant deltas.
2
+//!
3
+//! This stage sits between low-level token decoding and final block assembly.
4
+//! It drives one unified parser that may emit normal text, reasoning text, or
5
+//! tool-call deltas, then normalizes those parser events into internal
6
+//! assistant events.
7
+
8
+use asynk_strim_attr::{TryYielder, try_stream};
9
+use futures::{StreamExt as _, pin_mut};
10
+use thiserror_ext::AsReport;
11
+use tracing::warn;
12
+use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
13
+use vllm_text::output::DecodedTextEvent;
14
+
15
+use crate::Result;
16
+use crate::error::Error;
17
+use crate::event::AssistantBlockKind;
18
+use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id};
19
+
20
+/// Per-stream unified parsing state.
21
+struct UnifiedParserState {
22
+ /// Parser for the current request stream.
23
+ parser: Box<dyn UnifiedParser>,
24
+ /// Whether unified parsing has already failed for this stream.
25
+ parser_failed: bool,
26
+ /// The parser-local index of the currently open tool call, if any.
27
+ ///
28
+ /// Supported parsers currently emit at most one active tool call at a time.
29
+ /// Change this to an indexed map if a model needs interleaved calls later.
30
+ open_call_index: Option<usize>,
31
+}
32
+
33
+impl UnifiedParserState {
34
+ /// Create one fresh unified parsing state for a new streamed response.
35
+ fn new(parser: Box<dyn UnifiedParser>) -> Self {
36
+ Self {
37
+ parser,
38
+ parser_failed: false,
39
+ open_call_index: None,
40
+ }
41
+ }
42
+
43
+ /// Initialize parser state once prompt token IDs are available.
44
+ fn initialize(&mut self, prompt_token_ids: &[u32]) {
45
+ if self.parser_failed {
46
+ return;
47
+ }
48
+
49
+ match self.parser.initialize(prompt_token_ids) {
50
+ Ok(()) => {}
51
+ Err(error) => {
52
+ warn!(
53
+ error = %error.as_report(),
54
+ "failed to initialize unified parser; falling back to plain text deltas"
55
+ );
56
+ self.parser_failed = true;
57
+ self.open_call_index = None;
58
+ }
59
+ }
60
+ }
61
+
62
+ /// Convert one decoded text delta into zero or more parsed assistant events.
63
+ fn process_delta(&mut self, delta: String) -> Result<Vec<AssistantEvent>> {
64
+ if self.parser_failed {
65
+ self.open_call_index = None;
66
+ return Ok(text_event(AssistantBlockKind::Text, delta).into_iter().collect());
67
+ }
68
+
69
+ let mut output = UnifiedParserOutput::default();
70
+ match self.parser.parse_into(&delta, &mut output) {
71
+ Ok(()) => {
72
+ let mut events = Vec::new();
73
+ self.process_parser_output(output, &mut events)?;
74
+ Ok(events)
75
+ }
76
+ Err(error) => {
77
+ warn!(
78
+ error = %error.as_report(),
79
+ "unified parser failed; falling back to plain text deltas"
80
+ );
81
+ self.parser_failed = true;
82
+
83
+ let mut events = Vec::new();
84
+ self.process_parser_output(output, &mut events)?;
85
+ self.open_call_index = None;
86
+
87
+ let recovered = self.parser.reset();
88
+ if recovered.is_empty() && events.is_empty() {
89
+ push_text_delta(&mut events, AssistantBlockKind::Text, delta);
90
+ } else {
91
+ push_text_delta(&mut events, AssistantBlockKind::Text, recovered);
92
+ }
93
+ Ok(events)
94
+ }
95
+ }
96
+ }
97
+
98
+ /// Flush parser state at end-of-stream and close any remaining open calls.
99
+ fn finish(&mut self) -> Result<Vec<AssistantEvent>> {
100
+ let mut events = Vec::new();
101
+
102
+ if self.parser_failed {
103
+ return Ok(events);
104
+ }
105
+
106
+ match self.parser.finish() {
107
+ Ok(output) => self.process_parser_output(output, &mut events)?,
108
+ Err(error) => {
109
+ warn!(
110
+ error = %error.as_report(),
111
+ "unified parser finish failed; closing open parser state"
112
+ );
113
+ self.parser_failed = true;
114
+ self.open_call_index = None;
115
+ // TODO: should we reset and emit the buffered text?
116
+ }
117
+ }
118
+
119
+ Ok(events)
120
+ }
121
+
122
+ /// Apply one parsed unified output to the current stream state.
123
+ fn process_parser_output(
124
+ &mut self,
125
+ output: UnifiedParserOutput,
126
+ events: &mut Vec<AssistantEvent>,
127
+ ) -> Result<()> {
128
+ for event in output.events {
129
+ match event {
130
+ UnifiedParserEvent::Text(delta) => {
131
+ self.open_call_index = None;
132
+ push_text_delta(events, AssistantBlockKind::Text, delta);
133
+ }
134
+ UnifiedParserEvent::Reasoning(delta) => {
135
+ self.open_call_index = None;
136
+ push_text_delta(events, AssistantBlockKind::Reasoning, delta);
137
+ }
138
+ UnifiedParserEvent::ToolCall(item) => {
139
+ self.process_tool_item(item, events)?;
140
+ }
141
+ }
142
+ }
143
+
144
+ Ok(())
145
+ }
146
+
147
+ /// Apply one parsed tool-call delta emitted by the parser.
148
+ fn process_tool_item(
149
+ &mut self,
150
+ item: vllm_parser::tool::ToolCallDelta,
151
+ events: &mut Vec<AssistantEvent>,
152
+ ) -> Result<()> {
153
+ if let Some(name) = item.name {
154
+ let is_new_tool = match self.open_call_index {
155
+ Some(open_call_index) => open_call_index != item.tool_index,
156
+ None => true,
157
+ };
158
+ if is_new_tool {
159
+ let id = self
160
+ .parser
161
+ .tool_call_id(item.tool_index)
162
+ .map(str::to_string)
163
+ .unwrap_or_else(generate_tool_call_id);
164
+ self.open_call_index = Some(item.tool_index);
165
+ events.push(AssistantEvent::ToolCallStart { id, name });
166
+ }
167
+ }
168
+
169
+ if item.arguments.is_empty() {
170
+ return Ok(());
171
+ }
172
+ let Some(open_call_index) = self.open_call_index else {
173
+ return Err(Error::ToolCallStreamInvariant {
174
+ message: format!(
175
+ "received arguments for tool index {} before any tool-call start",
176
+ item.tool_index
177
+ ),
178
+ });
179
+ };
180
+ if open_call_index != item.tool_index {
181
+ return Err(Error::ToolCallStreamInvariant {
182
+ message: format!(
183
+ "received arguments for tool index {} while tool index {} is open",
184
+ item.tool_index, open_call_index
185
+ ),
186
+ });
187
+ }
188
+
189
+ events.push(AssistantEvent::ToolCallArgumentsDelta {
190
+ delta: item.arguments,
191
+ });
192
+ Ok(())
193
+ }
194
+}
195
+
196
+/// Build one plain text event if `delta` is non-empty.
197
+fn text_event(kind: AssistantBlockKind, delta: String) -> Option<AssistantEvent> {
198
+ if delta.is_empty() {
199
+ return None;
200
+ }
201
+ Some(AssistantEvent::TextDelta { kind, delta })
202
+}
203
+
204
+/// Push one plain text delta if it is non-empty.
205
+fn push_text_delta(events: &mut Vec<AssistantEvent>, kind: AssistantBlockKind, delta: String) {
206
+ if let Some(event) = text_event(kind, delta) {
207
+ events.push(event);
208
+ }
209
+}
210
+
211
+/// Wrap one decoded-text stream into the internal unified assistant stream.
212
+#[try_stream]
213
+pub(crate) async fn unified_event_stream(
214
+ decoded_stream: impl DecodedTextEventStream,
215
+ parser: Box<dyn UnifiedParser>,
216
+ mut y: TryYielder<AssistantEvent, Error>,
217
+) -> Result<()> {
218
+ pin_mut!(decoded_stream);
219
+
220
+ let mut state = UnifiedParserState::new(parser);
221
+
222
+ while let Some(event) = decoded_stream.next().await.transpose()? {
223
+ match event {
224
+ DecodedTextEvent::Start {
225
+ prompt_token_ids,
226
+ prompt_logprobs,
227
+ } => {
228
+ state.initialize(&prompt_token_ids);
229
+ y.yield_ok(AssistantEvent::Start {
230
+ prompt_token_ids,
231
+ prompt_logprobs,
232
+ })
233
+ .await;
234
+ }
235
+ DecodedTextEvent::TextDelta {
236
+ delta,
237
+ token_ids,
238
+ logprobs,
239
+ finished,
240
+ } => {
241
+ for next in state.process_delta(delta)? {
242
+ y.yield_ok(next).await;
243
+ }
244
+ if logprobs.is_some() || !token_ids.is_empty() {
245
+ y.yield_ok(AssistantEvent::LogprobsDelta {
246
+ logprobs,
247
+ token_ids,
248
+ })
249
+ .await;
250
+ }
251
+ if let Some(finished) = finished {
252
+ for next in state.finish()? {
253
+ y.yield_ok(next).await;
254
+ }
255
+ y.yield_ok(AssistantEvent::Done {
256
+ usage: finished.usage,
257
+ finish_reason: finished.finish_reason,
258
+ kv_transfer_params: finished.kv_transfer_params,
259
+ })
260
+ .await;
261
+ }
262
+ }
263
+ }
264
+ }
265
+ Ok(())
266
+}
267
+
268
+#[cfg(test)]
269
+mod tests {
270
+ use std::collections::VecDeque;
271
+
272
+ use futures::{StreamExt as _, stream};
273
+ use vllm_parser::reasoning::ReasoningError;
274
+ use vllm_parser::tool::ToolCallDelta;
275
+ use vllm_parser::unified::{UnifiedParserError, UnifiedParserOutput};
276
+
277
+ use super::unified_event_stream;
278
+ use crate::event::AssistantBlockKind;
279
+ use crate::output::AssistantEvent;
280
+
281
+ enum ScriptedStep {
282
+ Output(UnifiedParserOutput),
283
+ Error {
284
+ committed: UnifiedParserOutput,
285
+ reset_text: String,
286
+ },
287
+ }
288
+
289
+ struct ScriptedParser {
290
+ steps: VecDeque<ScriptedStep>,
291
+ reset_text: String,
292
+ tool_call_id: Option<String>,
293
+ finish_error_reset_text: Option<String>,
294
+ }
295
+
296
+ impl ScriptedParser {
297
+ fn new(steps: impl IntoIterator<Item = ScriptedStep>) -> Self {
298
+ Self {
299
+ steps: steps.into_iter().collect(),
300
+ reset_text: String::new(),
301
+ tool_call_id: Some("call_test".to_string()),
302
+ finish_error_reset_text: None,
303
+ }
304
+ }
305
+
306
+ fn with_finish_error(mut self, reset_text: &str) -> Self {
307
+ self.finish_error_reset_text = Some(reset_text.to_string());
308
+ self
309
+ }
310
+ }
311
+
312
+ impl vllm_parser::unified::UnifiedParser for ScriptedParser {
313
+ fn create(
314
+ _tools: &[vllm_parser::tool::Tool],
315
+ _tokenizer: vllm_tokenizer::DynTokenizer,
316
+ ) -> vllm_parser::unified::Result<Box<dyn vllm_parser::unified::UnifiedParser>>
317
+ where
318
+ Self: Sized + 'static,
319
+ {
320
+ unreachable!("ScriptedParser is constructed directly in tests")
321
+ }
322
+
323
+ fn parse_into(
324
+ &mut self,
325
+ _delta: &str,
326
+ output: &mut UnifiedParserOutput,
327
+ ) -> vllm_parser::unified::Result<()> {
328
+ match self.steps.pop_front().expect("unexpected parser call") {
329
+ ScriptedStep::Output(next) => {
330
+ output.append(next);
331
+ Ok(())
332
+ }
333
+ ScriptedStep::Error {
334
+ committed,
335
+ reset_text,
336
+ } => {
337
+ output.append(committed);
338
+ self.reset_text = reset_text;
339
+ Err(UnifiedParserError::Reasoning(
340
+ ReasoningError::MissingToken {
341
+ token: "<think>".to_string(),
342
+ },
343
+ ))
344
+ }
345
+ }
346
+ }
347
+
348
+ fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
349
+ self.tool_call_id.as_deref()
350
+ }
351
+
352
+ fn finish(&mut self) -> vllm_parser::unified::Result<UnifiedParserOutput> {
353
+ if let Some(reset_text) = self.finish_error_reset_text.take() {
354
+ self.reset_text = reset_text;
355
+ return Err(UnifiedParserError::Reasoning(
356
+ ReasoningError::MissingToken {
357
+ token: "</tool_call>".to_string(),
358
+ },
359
+ ));
360
+ }
361
+ Ok(UnifiedParserOutput::default())
362
+ }
363
+
364
+ fn reset(&mut self) -> String {
365
+ std::mem::take(&mut self.reset_text)
366
+ }
367
+ }
368
+
369
+ fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
370
+ vllm_text::output::DecodedTextEvent::TextDelta {
371
+ delta: delta.to_string(),
372
+ token_ids: Vec::new(),
373
+ logprobs: None,
374
+ finished: None,
375
+ }
376
+ }
377
+
378
+ fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent {
379
+ vllm_text::output::DecodedTextEvent::TextDelta {
380
+ delta: delta.to_string(),
381
+ token_ids: Vec::new(),
382
+ logprobs: None,
383
+ finished: Some(vllm_text::output::Finished {
384
+ usage: vllm_llm::TokenUsage::default(),
385
+ finish_reason: crate::FinishReason::Stop(None),
386
+ kv_transfer_params: None,
387
+ }),
388
+ }
389
+ }
390
+
391
+ async fn collect(
392
+ parser: ScriptedParser,
393
+ events: Vec<vllm_text::output::DecodedTextEvent>,
394
+ ) -> Vec<AssistantEvent> {
395
+ let stream = stream::iter(events.into_iter().map(Ok));
396
+ unified_event_stream(stream, Box::new(parser))
397
+ .collect::<Vec<_>>()
398
+ .await
399
+ .into_iter()
400
+ .collect::<crate::Result<Vec<_>>>()
401
+ .unwrap()
402
+ }
403
+
404
+ fn text(delta: &str) -> UnifiedParserOutput {
405
+ let mut output = UnifiedParserOutput::default();
406
+ output.push_text(delta.to_string());
407
+ output
408
+ }
409
+
410
+ fn reasoning(delta: &str) -> UnifiedParserOutput {
411
+ let mut output = UnifiedParserOutput::default();
412
+ output.push_reasoning(delta.to_string());
413
+ output
414
+ }
415
+
416
+ fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput {
417
+ UnifiedParserOutput {
418
+ events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall(
419
+ ToolCallDelta {
420
+ tool_index: 0,
421
+ name: Some(name.to_string()),
422
+ arguments: arguments.to_string(),
423
+ },
424
+ )],
425
+ }
426
+ }
427
+
428
+ fn combined(first: UnifiedParserOutput, second: UnifiedParserOutput) -> UnifiedParserOutput {
429
+ let mut output = first;
430
+ output.append(second);
431
+ output
432
+ }
433
+
434
+ #[tokio::test]
435
+ async fn unified_stream_emits_reasoning_only_deltas() {
436
+ let events = collect(
437
+ ScriptedParser::new([ScriptedStep::Output(reasoning("thinking"))]),
438
+ vec![decoded_delta("raw")],
439
+ )
440
+ .await;
441
+
442
+ assert_eq!(
443
+ events,
444
+ vec![AssistantEvent::TextDelta {
445
+ kind: AssistantBlockKind::Reasoning,
446
+ delta: "thinking".to_string(),
447
+ }]
448
+ );
449
+ }
450
+
451
+ #[tokio::test]
452
+ async fn unified_stream_emits_tool_only_deltas() {
453
+ let events = collect(
454
+ ScriptedParser::new([ScriptedStep::Output(tool_call(
455
+ "get_weather",
456
+ r#"{"location":"Paris"}"#,
457
+ ))]),
458
+ vec![decoded_delta("raw")],
459
+ )
460
+ .await;
461
+
462
+ assert_eq!(
463
+ events,
464
+ vec![
465
+ AssistantEvent::ToolCallStart {
466
+ id: "call_test".to_string(),
467
+ name: "get_weather".to_string(),
468
+ },
469
+ AssistantEvent::ToolCallArgumentsDelta {
470
+ delta: r#"{"location":"Paris"}"#.to_string(),
471
+ },
472
+ ]
473
+ );
474
+ }
475
+
476
+ #[tokio::test]
477
+ async fn unified_stream_emits_reasoning_followed_by_tool_call() {
478
+ let events = collect(
479
+ ScriptedParser::new([ScriptedStep::Output(combined(
480
+ reasoning("thinking"),
481
+ tool_call("get_weather", r#"{"location":"Paris"}"#),
482
+ ))]),
483
+ vec![decoded_delta("raw")],
484
+ )
485
+ .await;
486
+
487
+ assert_eq!(
488
+ events,
489
+ vec![
490
+ AssistantEvent::TextDelta {
491
+ kind: AssistantBlockKind::Reasoning,
492
+ delta: "thinking".to_string(),
493
+ },
494
+ AssistantEvent::ToolCallStart {
495
+ id: "call_test".to_string(),
496
+ name: "get_weather".to_string(),
497
+ },
498
+ AssistantEvent::ToolCallArgumentsDelta {
499
+ delta: r#"{"location":"Paris"}"#.to_string(),
500
+ },
501
+ ]
502
+ );
503
+ }
504
+
505
+ #[tokio::test]
506
+ async fn unified_stream_emits_visible_text_followed_by_tool_call() {
507
+ let events = collect(
508
+ ScriptedParser::new([ScriptedStep::Output(combined(
509
+ text("visible "),
510
+ tool_call("get_weather", r#"{"location":"Paris"}"#),
511
+ ))]),
512
+ vec![decoded_delta("raw")],
513
+ )
514
+ .await;
515
+
516
+ assert_eq!(
517
+ events,
518
+ vec![
519
+ AssistantEvent::TextDelta {
520
+ kind: AssistantBlockKind::Text,
521
+ delta: "visible ".to_string(),
522
+ },
523
+ AssistantEvent::ToolCallStart {
524
+ id: "call_test".to_string(),
525
+ name: "get_weather".to_string(),
526
+ },
527
+ AssistantEvent::ToolCallArgumentsDelta {
528
+ delta: r#"{"location":"Paris"}"#.to_string(),
529
+ },
530
+ ]
531
+ );
532
+ }
533
+
534
+ #[tokio::test]
535
+ async fn unified_stream_fallback_keeps_committed_output_and_disables_later_parsing() {
536
+ let events = collect(
537
+ ScriptedParser::new([ScriptedStep::Error {
538
+ committed: text("committed"),
539
+ reset_text: "buffered".to_string(),
540
+ }]),
541
+ vec![decoded_delta("bad"), decoded_delta("later")],
542
+ )
543
+ .await;
544
+
545
+ assert_eq!(
546
+ events,
547
+ vec![
548
+ AssistantEvent::TextDelta {
549
+ kind: AssistantBlockKind::Text,
550
+ delta: "committed".to_string(),
551
+ },
552
+ AssistantEvent::TextDelta {
553
+ kind: AssistantBlockKind::Text,
554
+ delta: "buffered".to_string(),
555
+ },
556
+ AssistantEvent::TextDelta {
557
+ kind: AssistantBlockKind::Text,
558
+ delta: "later".to_string(),
559
+ },
560
+ ]
561
+ );
562
+ }
563
+
564
+ #[tokio::test]
565
+ async fn unified_stream_finish_error_closes_parser_without_reset_text() {
566
+ let events = collect(
567
+ ScriptedParser::new([ScriptedStep::Output(UnifiedParserOutput::default())])
568
+ .with_finish_error("buffered"),
569
+ vec![finished_delta("")],
570
+ )
571
+ .await;
572
+
573
+ assert_eq!(
574
+ events,
575
+ vec![AssistantEvent::Done {
576
+ usage: vllm_llm::TokenUsage::default(),
577
+ finish_reason: crate::FinishReason::Stop(None),
578
+ kv_transfer_params: None,
579
+ }]
580
+ );
581
+ }
582
+}
rust/src/chat/src/output/mod.rs
+2
-52
@@ -2,7 +2,6 @@ use std::pin::Pin;
2
use std::sync::Arc;
3
4
use futures::Stream;
5
-use subenum::subenum;
5
use trait_set::trait_set;
6
use uuid::Uuid;
7
use vllm_llm::TokenUsage;
@@ -22,23 +21,19 @@ pub(crate) use harmony::validate_harmony_parser_overrides;
21
22
/// Internal assistant event before final assembly.
23
///
25
-/// - [`ContentEvent`]: subenum after reasoning parsing, carries only text content.
26
-/// - [`AssistantEvent`]: full event after tool parsing, adds tool-call variants.
27
-#[subenum(ContentEvent)]
24
+/// Unified parsing produces these events, and structured assembly consumes
25
+/// them to build public chat events.
26
#[derive(Debug, Clone, PartialEq)]
27
pub(crate) enum AssistantEvent {
30
- #[subenum(ContentEvent)]
28
Start {
29
prompt_token_ids: Arc<[u32]>,
30
prompt_logprobs: Option<DecodedPromptLogprobs>,
31
},
35
- #[subenum(ContentEvent)]
32
TextDelta {
33
kind: AssistantBlockKind,
34
delta: String,
35
},
36
/// Per-decoded-update sample metadata: logprobs and/or output token IDs.
41
- #[subenum(ContentEvent)]
37
LogprobsDelta {
38
logprobs: Option<DecodedLogprobs>,
39
token_ids: Vec<u32>,
@@ -48,7 +43,6 @@ pub(crate) enum AssistantEvent {
43
/// A delta for the arguments of the currently open tool call. Must follow a
44
/// `ToolCallStart`.
45
ToolCallArgumentsDelta { delta: String },
51
- #[subenum(ContentEvent)]
46
Done {
47
usage: TokenUsage,
48
finish_reason: FinishReason,
@@ -57,50 +51,6 @@ pub(crate) enum AssistantEvent {
51
},
52
}
53
60
-impl ContentEvent {
61
- /// Convert a [`DecodedTextEvent`] into one or more [`ContentEvent`] values
62
- /// by treating all text as plain (non-reasoning) content.
63
- fn from_decoded_plain_text(event: DecodedTextEvent) -> Vec<Self> {
64
- match event {
65
- DecodedTextEvent::Start {
66
- prompt_token_ids,
67
- prompt_logprobs,
68
- } => vec![Self::Start {
69
- prompt_token_ids,
70
- prompt_logprobs,
71
- }],
72
- DecodedTextEvent::TextDelta {
73
- delta,
74
- token_ids,
75
- logprobs,
76
- finished,
77
- } => {
78
- let mut events = Vec::new();
79
- if !delta.is_empty() {
80
- events.push(Self::TextDelta {
81
- kind: AssistantBlockKind::Text,
82
- delta,
83
- });
84
- }
85
- if logprobs.is_some() || !token_ids.is_empty() {
86
- events.push(Self::LogprobsDelta {
87
- logprobs,
88
- token_ids,
89
- });
90
- }
91
- if let Some(finished) = finished {
92
- events.push(Self::Done {
93
- usage: finished.usage,
94
- finish_reason: finished.finish_reason,
95
- kv_transfer_params: finished.kv_transfer_params,
96
- });
97
- }
98
- events
99
- }
100
- }
101
- }
102
-}
103
-
54
/// Boxed stream of decoded text events coming from [`vllm_text`].
55
pub type DynDecodedTextEventStream = Pin<Box<dyn Stream<Item = Result<DecodedTextEvent>> + Send>>;
56
/// Boxed stream of structured chat events exposed by [`crate::ChatLlm`].
rust/src/chat/src/parser/reasoning/mod.rs
+2
-2
@@ -2,7 +2,7 @@
2
3
use std::sync::LazyLock;
4
5
-pub use vllm_reasoning_parser::{
5
+pub use vllm_parser::reasoning::{
6
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
7
DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
8
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
@@ -34,7 +34,7 @@ pub mod names {
34
35
/// Constructor signature for one registered reasoning parser implementation.
36
type ReasoningParserCreator =
37
- fn(DynTokenizer) -> vllm_reasoning_parser::Result<Box<dyn ReasoningParser>>;
37
+ fn(DynTokenizer) -> vllm_parser::reasoning::Result<Box<dyn ReasoningParser>>;
38
39
/// Registry and model matcher for reasoning parsers.
40
pub type ReasoningParserFactory = ParserFactory<ReasoningParserCreator>;
rust/src/chat/src/parser/tool/mod.rs
+3
-4
@@ -2,13 +2,12 @@
2
3
use std::sync::LazyLock;
4
5
-pub use vllm_tool_parser::{
5
+pub use vllm_parser::tool::{
6
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
7
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
8
HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
9
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
10
- Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError,
11
- ToolParserOutput,
10
+ Qwen3CoderToolParser, Qwen3XmlToolParser, ToolParser, ToolParserError,
11
};
12
13
use crate::parser::ParserFactory;
@@ -41,7 +40,7 @@ pub mod names {
40
}
41
42
/// Constructor signature for one registered tool parser implementation.
44
-type ToolParserCreator = fn(&[ChatTool]) -> vllm_tool_parser::Result<Box<dyn ToolParser>>;
43
+type ToolParserCreator = fn(&[ChatTool]) -> vllm_parser::tool::Result<Box<dyn ToolParser>>;
44
45
/// Registry and model matcher for tool parsers.
46
pub type ToolParserFactory = ParserFactory<ToolParserCreator>;
rust/src/chat/src/parser/tool/tests.rs
+2
-2
@@ -1,6 +1,6 @@
1
-use vllm_tool_parser::Result;
1
+use vllm_parser::tool::{Result, ToolParserOutput};
2
3
-use super::{ToolParser, ToolParserFactory, ToolParserOutput, names};
3
+use super::{ToolParser, ToolParserFactory, names};
4
use crate::Error;
5
use crate::request::ChatTool;
6
rust/src/chat/src/request.rs
+1
-1
@@ -4,9 +4,9 @@ use llm_multimodal::ImageDetail;
4
use serde::{Deserialize, Serialize};
5
use serde_json::Value;
6
use vllm_engine_core_client::protocol::lora::LoraRequest;
7
+pub use vllm_parser::tool::Tool as ChatTool;
8
pub use vllm_text::SamplingParams;
9
use vllm_text::TextDecodeOptions;
9
-pub use vllm_tool_parser::Tool as ChatTool;
10
11
use crate::AssistantMessageExt;
12
use crate::error::{Error, Result};
rust/src/chat/tests/chat.rs
+1
-1
@@ -1180,7 +1180,7 @@ async fn chat_stream_parses_tool_calls_automatically() {
1180
),
1181
request_output(
1182
"chat-tool",
1183
- bytes_to_token_ids(
1183
+ bytes_with_special_stop_token(
1184
b"\"arguments\":{\"city\":\"Paris\"}}\n</tool_call>",
1185
),
1186
Some(EngineCoreFinishReason::Stop),
rust/src/parser/Cargo.toml
renamed
+2
-1
@@ -1,5 +1,5 @@
1
[package]
2
-name = "vllm-tool-parser"
2
+name = "vllm-parser"
3
version.workspace = true
4
edition.workspace = true
5
license.workspace = true
@@ -13,6 +13,7 @@ serde.workspace = true
13
serde_json.workspace = true
14
thiserror.workspace = true
15
thiserror-ext.workspace = true
16
+vllm-tokenizer.workspace = true
17
winnow.workspace = true
18
xgrammar-structural-tag.workspace = true
19
rust/src/parser/benches/deepseek_v3.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::DeepSeekParser as ExternalDeepSeekParser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{DeepSeekV3ToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{DeepSeekV3ToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/deepseek_v31.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::DeepSeek31Parser as ExternalDeepSeek31Parser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{DeepSeekV31ToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{DeepSeekV31ToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/deepseek_v32.rs
renamed
+2
-2
@@ -1,8 +1,8 @@
1
use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
5
-use vllm_tool_parser::{DeepSeekV32ToolParser, Tool, ToolParser};
4
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
5
+use vllm_parser::tool::{DeepSeekV32ToolParser, Tool, ToolParser};
6
7
mod utils;
8
use utils::feed_parser;
rust/src/parser/benches/gemma4.rs
renamed
+2
-2
@@ -1,8 +1,8 @@
1
use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
5
-use vllm_tool_parser::{Gemma4ToolParser, Tool, ToolParser};
4
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
5
+use vllm_parser::tool::{Gemma4ToolParser, Tool, ToolParser};
6
7
mod utils;
8
use utils::feed_parser;
rust/src/parser/benches/glm45_moe.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::Glm4MoeParser as ExternalGlm4MoeParser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/kimi_k2.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::KimiK2Parser as ExternalKimiK2Parser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{KimiK2ToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{KimiK2ToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/llama3_json.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::LlamaParser as ExternalLlamaParser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{Llama3JsonToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{Llama3JsonToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/minimax_m2.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::MinimaxM2Parser as ExternalMinimaxM2Parser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{MinimaxM2ToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{MinimaxM2ToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/qwen3_coder.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{Qwen3CoderToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/qwen3_xml.rs
renamed
+2
-2
@@ -2,8 +2,8 @@ use std::time::Duration;
2
3
use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
4
use tool_parser::parsers::QwenParser as ExternalQwenParser;
5
-use vllm_tool_parser::test_utils::{split_by_chars, test_tools};
6
-use vllm_tool_parser::{Qwen3XmlToolParser, Tool, ToolParser};
5
+use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
6
+use vllm_parser::tool::{Qwen3XmlToolParser, Tool, ToolParser};
7
8
mod utils;
9
use utils::{feed_external_parser, feed_parser, openai_tools};
rust/src/parser/benches/utils/mod.rs
renamed
+2
-2
@@ -3,8 +3,8 @@
3
use futures::FutureExt as _;
4
use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool};
5
use tool_parser::traits::ToolParser as ExternalToolParser;
6
-use vllm_tool_parser::test_utils::collect_stream;
7
-use vllm_tool_parser::{Tool, ToolParser};
6
+use vllm_parser::tool::test_utils::collect_stream;
7
+use vllm_parser::tool::{Tool, ToolParser};
8
9
pub(super) fn openai_tools(tools: &[Tool]) -> Vec<OpenAiTool> {
10
tools
rust/src/parser/python/Cargo.toml
renamed
+1
-1
@@ -13,7 +13,7 @@ pyo3.workspace = true
13
pythonize = { workspace = true, features = ["serde_json"] }
14
serde_json.workspace = true
15
thiserror-ext.workspace = true
16
-vllm-tool-parser.workspace = true
16
+vllm-parser.workspace = true
17
18
[lints]
19
workspace = true
rust/src/parser/python/src/lib.rs
renamed
+3
-3
@@ -1,4 +1,4 @@
1
-//! Thin PyO3 bindings for `vllm_tool_parser`.
1
+//! Thin PyO3 bindings for `vllm_parser::tool`.
2
//!
3
//! This crate exposes the Rust tool parser trait and data shapes to Python
4
//! while keeping parser state, grammar, and schema-aware argument conversion in
@@ -11,7 +11,7 @@ use pyo3::types::{PyAny, PyModule};
11
use pythonize::{depythonize, pythonize};
12
use serde_json::Value;
13
use thiserror_ext::AsReport as _;
14
-use vllm_tool_parser::{Tool, ToolCallDelta, ToolParser, ToolParserOutput};
14
+use vllm_parser::tool::{Tool, ToolCallDelta, ToolParser, ToolParserOutput};
15
16
macro_rules! tool_parser_factory {
17
($($parser:ident),+ $(,)?) => {
@@ -22,7 +22,7 @@ macro_rules! tool_parser_factory {
22
match name {
23
$(
24
stringify!($parser) => {
25
- <vllm_tool_parser::$parser as ToolParser>::create(tools)
25
+ <vllm_parser::tool::$parser as ToolParser>::create(tools)
26
}
27
)+
28
_ => {
rust/src/parser/src/lib.rs
new
+5
@@ -0,0 +1,5 @@
1
+//! Streaming parsers for chat completions.
2
+
3
+pub mod reasoning;
4
+pub mod tool;
5
+pub mod unified;
rust/src/parser/src/reasoning/cohere_cmd.rs
renamed
rust/src/parser/src/reasoning/deepseek_r1.rs
renamed
rust/src/parser/src/reasoning/delimited.rs
renamed
rust/src/parser/src/reasoning/gemma4.rs
renamed
+1
-1
@@ -119,7 +119,7 @@ mod tests {
119
use vllm_tokenizer::Tokenizer;
120
121
use super::Gemma4ReasoningParser;
122
- use crate::ReasoningParser;
122
+ use crate::reasoning::ReasoningParser;
123
124
struct FakeTokenizer;
125
rust/src/parser/src/reasoning/kimi.rs
renamed
rust/src/parser/src/reasoning/minimax_m3.rs
renamed
rust/src/parser/src/reasoning/mod.rs
renamed
rust/src/parser/src/reasoning/qwen3.rs
renamed
rust/src/parser/src/reasoning/seed_oss.rs
renamed
+1
-1
@@ -49,7 +49,7 @@ mod tests {
49
use std::sync::Arc;
50
51
use super::SeedOssReasoningParser;
52
- use crate::{ReasoningParser, tests::FakeTokenizer};
52
+ use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
53
54
#[test]
55
fn without_prompt_markers_expects_start_token() {
rust/src/parser/src/reasoning/step3p5.rs
renamed
+1
-1
@@ -127,7 +127,7 @@ mod tests {
127
use std::sync::Arc;
128
129
use super::Step3p5ReasoningParser;
130
- use crate::{ReasoningParser, tests::FakeTokenizer};
130
+ use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
131
132
#[test]
133
fn picks_up_prompt_start_boundary() {
rust/src/parser/src/reasoning/tests.rs
renamed
rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{DeepSeekDsmlToolParser, DsmlTokens};
2
-use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4
/// Tool parser for DeepSeek V3.2 models.
5
///
@@ -67,8 +67,8 @@ mod tests {
67
use thiserror_ext::AsReport;
68
69
use super::DeepSeekV32ToolParser;
70
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
71
- use crate::{ToolParser, ToolParserTestExt as _};
70
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
71
+ use crate::tool::{ToolParser, ToolParserTestExt as _};
72
73
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
74
let params = params
rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{DeepSeekDsmlToolParser, DsmlTokens};
2
-use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4
/// Tool parser for DeepSeek V4 models.
5
///
@@ -69,8 +69,8 @@ mod tests {
69
use serde_json::{Value, json};
70
71
use super::DeepSeekV4ToolParser;
72
- use crate::test_utils::{collect_stream, test_tools};
73
- use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
72
+ use crate::tool::test_utils::{collect_stream, test_tools};
73
+ use crate::tool::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
74
75
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
76
let params = params
rust/src/parser/src/tool/deepseek_dsml/mod.rs
renamed
+1
-1
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
7
use super::parameters::ToolSchemas;
8
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9
use super::{Result, ToolCallDelta, ToolParserOutput};
10
-use crate::Tool;
10
+use crate::tool::Tool;
11
12
mod deepseek_v32;
13
mod deepseek_v4;
rust/src/parser/src/tool/deepseek_json/deepseek_v3.rs
renamed
+4
-4
@@ -1,5 +1,5 @@
1
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
2
-use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4
/// Tool parser for DeepSeek V3 JSON-fenced tool calls.
5
///
@@ -55,12 +55,12 @@ mod tests {
55
use thiserror_ext::AsReport;
56
57
use super::DeepSeekV3ToolParser;
58
- use crate::deepseek_json::{
58
+ use crate::tool::deepseek_json::{
59
TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START, V3_ARGUMENT_END,
60
V3_JSON_START,
61
};
62
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
63
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
62
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
63
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
64
65
fn v3_tool_call(function_name: &str, arguments: &str) -> String {
66
format!(
rust/src/parser/src/tool/deepseek_json/deepseek_v31.rs
renamed
+4
-4
@@ -1,5 +1,5 @@
1
use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
2
-use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4
/// Tool parser for DeepSeek V3.1 raw JSON tool calls.
5
///
@@ -51,11 +51,11 @@ mod tests {
51
use thiserror_ext::AsReport;
52
53
use super::DeepSeekV31ToolParser;
54
- use crate::deepseek_json::{
54
+ use crate::tool::deepseek_json::{
55
TOOL_CALL_END, TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START,
56
};
57
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
58
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
57
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
58
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
59
60
fn v31_tool_call(function_name: &str, arguments: &str) -> String {
61
format!("{TOOL_CALL_START}{function_name}{TOOL_CALL_SEPARATOR}{arguments}{TOOL_CALL_END}")
rust/src/parser/src/tool/deepseek_json/mod.rs
renamed
rust/src/parser/src/tool/error.rs
renamed
+1
-1
@@ -6,7 +6,7 @@ pub type Result<T> = std::result::Result<T, ToolParserError>;
6
7
/// Errors produced while creating or running tool parsers.
8
#[derive(Debug, Error, Macro)]
9
-#[thiserror_ext(macro(path = "crate::error"))]
9
+#[thiserror_ext(macro(path = "crate::tool::error"))]
10
pub enum ToolParserError {
11
#[error("tool parser parsing failed: {message}")]
12
ParsingFailed { message: String },
rust/src/parser/src/tool/gemma4.rs
renamed
+2
-2
@@ -8,7 +8,7 @@ use winnow::token::{literal, take_till, take_until};
8
9
use super::utils::{incomplete, parse_buffered_event, partial_prefix_len, safe_text_len};
10
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
11
-use crate::Tool;
11
+use crate::tool::Tool;
12
13
const TOOL_CALL_START: &str = "<|tool_call>";
14
const TOOL_CALL_END: &str = "<tool_call|>";
@@ -428,7 +428,7 @@ mod tests {
428
Gemma4ToolParser, ToolCallDelta, ToolParser, ToolParserOutput, gemma4_array_content,
429
parse_gemma4_args,
430
};
431
- use crate::{Tool, ToolParserTestExt as _};
431
+ use crate::tool::{Tool, ToolParserTestExt as _};
432
433
fn parse_gemma4_array(array: &str) -> super::Result<Vec<Value>> {
434
let mut input = array;
rust/src/parser/src/tool/glm_xml/glm45_moe.rs
renamed
+1
-1
@@ -1,5 +1,5 @@
1
use super::{GlmXmlToolParser, Separator};
2
-use crate::{Result, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
3
4
/// Tool parser for GLM-4.5/4.6 MoE XML-style tool calls.
5
///
rust/src/parser/src/tool/glm_xml/glm47_moe.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{GlmXmlToolParser, Separator};
2
-use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
5
///
@@ -44,8 +44,8 @@ mod tests {
44
use serde_json::{Value, json};
45
46
use super::Glm47MoeToolParser;
47
- use crate::ToolParserTestExt as _;
48
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
47
+ use crate::tool::ToolParserTestExt as _;
48
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
49
50
fn glm47_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
51
let params = params
rust/src/parser/src/tool/glm_xml/mod.rs
renamed
+3
-3
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until, take_while};
7
use super::parameters::ToolSchemas;
8
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9
use super::{Result, ToolCallDelta, ToolParserOutput};
10
-use crate::Tool;
10
+use crate::tool::Tool;
11
12
mod glm45_moe;
13
mod glm47_moe;
@@ -264,8 +264,8 @@ mod tests {
264
use thiserror_ext::AsReport;
265
266
use super::Glm45MoeToolParser;
267
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
268
- use crate::{ToolParser, ToolParserTestExt as _};
267
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
268
+ use crate::tool::{ToolParser, ToolParserTestExt as _};
269
270
fn glm45_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
271
let params = params
rust/src/parser/src/tool/hy_v3.rs
renamed
+3
-3
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
7
use super::parameters::ToolSchemas;
8
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
10
-use crate::{StructuralTagModel, Tool};
10
+use crate::tool::{StructuralTagModel, Tool};
11
12
const TOOL_CALLS_START: &str = "<tool_calls>";
13
const TOOL_CALLS_END: &str = "</tool_calls>";
@@ -249,8 +249,8 @@ mod tests {
249
use thiserror_ext::AsReport;
250
251
use super::{HyV3ToolParser, ToolParser};
252
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
253
- use crate::{ToolParserOutput, ToolParserTestExt as _};
252
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
253
+ use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
254
255
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
256
let params = params
rust/src/parser/src/tool/json/granite4.rs
renamed
+4
-4
@@ -8,10 +8,10 @@ use super::{
8
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
9
tool_call_header_event,
10
};
11
-use crate::utils::{
11
+use crate::tool::utils::{
12
JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, take_json_object,
13
};
14
-use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
14
+use crate::tool::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
15
16
const TOOL_CALL_START: &str = "<tool_call>";
17
const TOOL_CALL_END: &str = "</tool_call>";
@@ -279,8 +279,8 @@ mod tests {
279
use thiserror_ext::AsReport;
280
281
use super::Granite4ToolParser;
282
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
283
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
282
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
283
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
284
285
#[test]
286
fn granite4_parse_complete_without_tool_call_keeps_text() {
rust/src/parser/src/tool/json/hermes.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2
-use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4
const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5
parser_name: "Hermes",
@@ -68,8 +68,8 @@ mod tests {
68
use thiserror_ext::AsReport;
69
70
use super::HermesToolParser;
71
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
72
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
71
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
72
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
73
74
fn build_tool_call(function_name: &str, arguments: &str) -> String {
75
format!(r#"<tool_call>{{"name":"{function_name}","arguments":{arguments}}}</tool_call>"#)
rust/src/parser/src/tool/json/internlm2.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2
-use crate::{Result, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
3
4
const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5
parser_name: "InternLM2",
@@ -123,8 +123,8 @@ mod tests {
123
use thiserror_ext::AsReport;
124
125
use super::Internlm2ToolParser;
126
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
127
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
126
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
127
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
128
129
const ACTION_START: &str = "<|action_start|><|plugin|>";
130
const ACTION_END: &str = "<|action_end|>";
rust/src/parser/src/tool/json/llama.rs
renamed
+4
-4
@@ -8,8 +8,8 @@ use super::{
8
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
9
argument_delta_event, tool_call_header_event,
10
};
11
-use crate::utils::{JsonObjectScanState, parse_buffered_event};
12
-use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
11
+use crate::tool::utils::{JsonObjectScanState, parse_buffered_event};
12
+use crate::tool::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
13
14
#[derive(Debug, Clone, PartialEq, Eq)]
15
enum LlamaJsonMode {
@@ -256,8 +256,8 @@ mod tests {
256
use thiserror_ext::AsReport;
257
258
use super::Llama3JsonToolParser;
259
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
260
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
259
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
260
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
261
262
fn build_tool_call(function_name: &str, parameters: &str) -> String {
263
format!(r#"{{"name":"{function_name}","parameters":{parameters}}}"#)
rust/src/parser/src/tool/json/mistral.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2
-use crate::{Result, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
3
4
const MISTRAL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5
parser_name: "Mistral",
@@ -61,8 +61,8 @@ mod tests {
61
use thiserror_ext::AsReport;
62
63
use super::MistralToolParser;
64
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
65
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
64
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
65
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
66
67
fn build_tool_call(function_name: &str, arguments: &str) -> String {
68
format!(r#"{{"name":"{function_name}","arguments":{arguments}}}"#)
rust/src/parser/src/tool/json/mod.rs
renamed
+1
-1
@@ -374,7 +374,7 @@ mod tests {
374
use expect_test::expect;
375
376
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
377
- use crate::ToolParserOutput;
377
+ use crate::tool::ToolParserOutput;
378
379
const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
380
parser_name: "Delimited JSON",
rust/src/parser/src/tool/json/phi4mini.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2
-use crate::{Result, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
3
4
const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5
parser_name: "Phi4Mini",
@@ -69,8 +69,8 @@ mod tests {
69
use thiserror_ext::AsReport;
70
71
use super::Phi4MiniJsonToolParser;
72
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
73
- use crate::{ToolParser, ToolParserTestExt as _};
72
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
73
+ use crate::tool::{ToolParser, ToolParserTestExt as _};
74
75
/// Build one phi-4-mini tool-call object: `{"name":..,"<args_key>":<args>}`.
76
fn build_call(function_name: &str, args_key: &str, arguments: &str) -> String {
rust/src/parser/src/tool/json/qwen.rs
renamed
+3
-3
@@ -1,5 +1,5 @@
1
use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2
-use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
2
+use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4
const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5
parser_name: "Qwen XML",
@@ -70,8 +70,8 @@ mod tests {
70
use thiserror_ext::AsReport;
71
72
use super::Qwen3XmlToolParser;
73
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
74
- use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
73
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
74
+ use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
75
76
fn build_tool_call(function_name: &str, arguments: &str) -> String {
77
format!(
rust/src/parser/src/tool/kimi_k2.rs
renamed
+3
-3
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while};
8
9
use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object};
10
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
11
-use crate::{StructuralTagModel, Tool};
11
+use crate::tool::{StructuralTagModel, Tool};
12
13
const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>";
14
const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>";
@@ -339,8 +339,8 @@ mod tests {
339
KimiK2ToolParser, TOOL_CALL_ARGUMENT_START, TOOL_CALL_END, TOOL_CALL_START, TOOL_CALLS_END,
340
TOOL_CALLS_START, ToolParser, tool_header,
341
};
342
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
343
- use crate::{ToolParserOutput, ToolParserTestExt as _};
342
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
343
+ use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
344
345
fn build_tool_call(function_name: &str, index: usize, arguments: &str) -> String {
346
format!(
rust/src/parser/src/tool/minimax_m2.rs
renamed
+3
-3
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
7
use super::parameters::ToolSchemas;
8
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
10
-use crate::{StructuralTagModel, Tool};
10
+use crate::tool::{StructuralTagModel, Tool};
11
12
const TOOL_CALL_START: &str = "<minimax:tool_call>";
13
const TOOL_CALL_END: &str = "</minimax:tool_call>";
@@ -272,8 +272,8 @@ mod tests {
272
use thiserror_ext::AsReport;
273
274
use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser};
275
- use crate::ToolParserTestExt as _;
276
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
275
+ use crate::tool::ToolParserTestExt as _;
276
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
277
278
fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String {
279
let invokes = invokes
rust/src/parser/src/tool/minimax_m3.rs
renamed
+3
-3
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until};
8
use super::parameters::{ParamElement, ParamInput, ToolSchemas};
9
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
10
use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
11
-use crate::Tool;
11
+use crate::tool::Tool;
12
13
const NAMESPACE: &str = "]<]minimax[>[";
14
const TOOL_CALL_START: &str = "]<]minimax[>[<tool_call>";
@@ -388,8 +388,8 @@ mod tests {
388
ELEMENT_END_START, ELEMENT_START, INVOKE_END, INVOKE_START, MinimaxM3ToolParser,
389
TOOL_CALL_END, TOOL_CALL_START, ToolParser,
390
};
391
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
392
- use crate::{Tool, ToolParserTestExt as _};
391
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
392
+ use crate::tool::{Tool, ToolParserTestExt as _};
393
394
fn element(name: &str, body: &str) -> String {
395
format!("{ELEMENT_START}{name}>{body}{ELEMENT_END_START}{name}>")
rust/src/parser/src/tool/mod.rs
renamed
+3
-3
@@ -1,9 +1,9 @@
1
//! Streaming tool parsers for chat completions.
2
3
#[macro_use]
4
-mod error;
4
+pub(crate) mod error;
5
mod deepseek_dsml;
6
-mod deepseek_json;
6
+pub(crate) mod deepseek_json;
7
mod gemma4;
8
mod glm_xml;
9
mod hy_v3;
@@ -15,7 +15,7 @@ mod parameters;
15
mod qwen_coder;
16
#[cfg(any(test, feature = "test-util"))]
17
pub mod test_utils;
18
-mod utils;
18
+pub(crate) mod utils;
19
20
use std::collections::{BTreeMap, btree_map};
21
rust/src/parser/src/tool/parameters.rs
renamed
+2
-2
@@ -2,7 +2,7 @@ use std::collections::BTreeMap;
2
3
use serde_json::{Map, Number, Value};
4
5
-use crate::Tool;
5
+use crate::tool::Tool;
6
7
/// Normalized parameter schemas for all tools in one request.
8
#[derive(Debug, Clone, Default, PartialEq, Eq)]
@@ -416,7 +416,7 @@ mod tests {
416
use serde_json::{Value, json};
417
418
use super::{ParamElement, ParamInput, ToolSchema, ToolSchemas};
419
- use crate::Tool;
419
+ use crate::tool::Tool;
420
421
fn test_tool(name: &str, parameters: serde_json::Value) -> Tool {
422
Tool {
rust/src/parser/src/tool/qwen_coder.rs
renamed
+3
-3
@@ -7,7 +7,7 @@ use winnow::token::{literal, take_until};
7
use super::parameters::ToolSchemas;
8
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9
use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput};
10
-use crate::Tool;
10
+use crate::tool::Tool;
11
12
const TOOL_CALL_START: &str = "<tool_call>";
13
const TOOL_CALL_END: &str = "</tool_call>";
@@ -241,8 +241,8 @@ mod tests {
241
use thiserror_ext::AsReport;
242
243
use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser};
244
- use crate::test_utils::{collect_stream, split_by_chars, test_tools};
245
- use crate::{ToolParserOutput, ToolParserTestExt as _};
244
+ use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
245
+ use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
246
247
fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
248
let params = params
rust/src/parser/src/tool/test_utils.rs
renamed
+1
-1
@@ -1,7 +1,7 @@
1
use serde_json::json;
2
3
use super::{ToolParser, ToolParserOutput};
4
-use crate::{Tool, ToolParserTestExt as _};
4
+use crate::tool::{Tool, ToolParserTestExt as _};
5
6
/// Build a reusable set of function tools for parser unit tests.
7
pub fn test_tools() -> Vec<Tool> {
rust/src/parser/src/tool/tests.rs
renamed
+1
-1
@@ -1,5 +1,5 @@
1
use super::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
2
-use crate::ToolParserTestExt as _;
2
+use crate::tool::ToolParserTestExt as _;
3
4
struct DefaultParser;
5
rust/src/parser/src/tool/utils.rs
renamed
rust/src/parser/src/unified/combined.rs
new
+346
@@ -0,0 +1,346 @@
1
+//! Adapter that combines reasoning and tool parsers.
2
+
3
+use vllm_tokenizer::DynTokenizer;
4
+
5
+use crate::reasoning::ReasoningParser;
6
+use crate::tool::{StructuralTagModel, Tool, ToolParser, ToolParserOutput};
7
+
8
+use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
9
+
10
+/// Unified parser that composes existing reasoning and tool parsers.
11
+pub struct CombinedParser {
12
+ reasoning: Option<Box<dyn ReasoningParser>>,
13
+ tool: Option<Box<dyn ToolParser>>,
14
+}
15
+
16
+impl CombinedParser {
17
+ /// Create a combined parser from optional reasoning and tool parsers.
18
+ pub fn new(
19
+ reasoning: Option<Box<dyn ReasoningParser>>,
20
+ tool: Option<Box<dyn ToolParser>>,
21
+ ) -> Self {
22
+ Self { reasoning, tool }
23
+ }
24
+
25
+ /// Create a text-only combined parser.
26
+ pub fn plain_text_only() -> Self {
27
+ Self {
28
+ reasoning: None,
29
+ tool: None,
30
+ }
31
+ }
32
+
33
+ fn parse_tool(&mut self, content: &str, output: &mut UnifiedParserOutput) -> Result<()> {
34
+ let Some(tool) = self.tool.as_mut() else {
35
+ output.push_text(content.to_string());
36
+ return Ok(());
37
+ };
38
+
39
+ // Preserve any tool output that was already produced before the error.
40
+ let mut tool_output = ToolParserOutput::default();
41
+ let result = tool.parse_into(content, &mut tool_output);
42
+ output.append_tool_output(tool_output);
43
+ result?;
44
+
45
+ Ok(())
46
+ }
47
+
48
+ fn flush_tool(&mut self) -> Result<UnifiedParserOutput> {
49
+ let Some(tool) = self.tool.as_mut() else {
50
+ return Ok(UnifiedParserOutput::default());
51
+ };
52
+
53
+ let output = tool.finish()?;
54
+ let mut unified = UnifiedParserOutput::default();
55
+ unified.append_tool_output(output);
56
+ Ok(unified)
57
+ }
58
+}
59
+
60
+impl UnifiedParser for CombinedParser {
61
+ fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
62
+ where
63
+ Self: Sized + 'static,
64
+ {
65
+ Err(UnifiedParserError::CombinedParserConstructor)
66
+ }
67
+
68
+ fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
69
+ if let Some(reasoning) = self.reasoning.as_mut() {
70
+ reasoning.initialize(prompt_token_ids)?;
71
+ }
72
+ Ok(())
73
+ }
74
+
75
+ fn preserve_special_tokens(&self) -> bool {
76
+ self.reasoning.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
77
+ || self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
78
+ }
79
+
80
+ fn structural_tag_model(&self) -> Option<StructuralTagModel> {
81
+ self.tool.as_ref().and_then(|parser| parser.structural_tag_model())
82
+ }
83
+
84
+ fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
85
+ self.tool.as_ref().and_then(|parser| parser.tool_call_id(tool_index))
86
+ }
87
+
88
+ fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> {
89
+ let Some(reasoning) = self.reasoning.as_mut() else {
90
+ return self.parse_tool(delta, output);
91
+ };
92
+
93
+ let reasoning_delta = reasoning.push(delta)?;
94
+ if let Some(reasoning) = reasoning_delta.reasoning {
95
+ output.push_reasoning(reasoning);
96
+ }
97
+ if let Some(content) = reasoning_delta.content {
98
+ self.parse_tool(&content, output)?;
99
+ }
100
+ Ok(())
101
+ }
102
+
103
+ fn finish(&mut self) -> Result<UnifiedParserOutput> {
104
+ let mut output = UnifiedParserOutput::default();
105
+ if let Some(reasoning) = self.reasoning.as_mut() {
106
+ let reasoning_delta = reasoning.finish()?;
107
+ if let Some(reasoning) = reasoning_delta.reasoning {
108
+ output.push_reasoning(reasoning);
109
+ }
110
+ if let Some(content) = reasoning_delta.content {
111
+ self.parse_tool(&content, &mut output)?;
112
+ }
113
+ }
114
+ output.append(self.flush_tool()?);
115
+ Ok(output)
116
+ }
117
+
118
+ fn reset(&mut self) -> String {
119
+ self.tool.as_mut().map_or_else(String::new, |parser| parser.reset())
120
+ }
121
+}
122
+
123
+#[cfg(test)]
124
+mod tests {
125
+ use std::sync::Arc;
126
+
127
+ use vllm_tokenizer::Tokenizer;
128
+
129
+ use super::CombinedParser;
130
+ use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};
131
+ use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};
132
+ use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
133
+
134
+ struct FakeTokenizer;
135
+
136
+ impl Tokenizer for FakeTokenizer {
137
+ fn encode(
138
+ &self,
139
+ text: &str,
140
+ _add_special_tokens: bool,
141
+ ) -> vllm_tokenizer::Result<Vec<u32>> {
142
+ Ok(text.chars().map(u32::from).collect())
143
+ }
144
+
145
+ fn decode(
146
+ &self,
147
+ token_ids: &[u32],
148
+ _skip_special_tokens: bool,
149
+ ) -> vllm_tokenizer::Result<String> {
150
+ Ok(token_ids
151
+ .iter()
152
+ .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
153
+ .collect())
154
+ }
155
+
156
+ fn token_to_id(&self, token: &str) -> Option<u32> {
157
+ match token {
158
+ "<think>" => Some(1),
159
+ "</think>" => Some(2),
160
+ _ => None,
161
+ }
162
+ }
163
+ }
164
+
165
+ fn test_tools() -> Vec<Tool> {
166
+ vec![Tool {
167
+ name: "get_weather".to_string(),
168
+ description: None,
169
+ parameters: serde_json::json!({
170
+ "type": "object",
171
+ "properties": {
172
+ "location": { "type": "string" }
173
+ },
174
+ }),
175
+ strict: None,
176
+ }]
177
+ }
178
+
179
+ fn collect(parser: &mut dyn UnifiedParser, chunks: &[&str]) -> UnifiedParserOutput {
180
+ let mut output = UnifiedParserOutput::default();
181
+ for chunk in chunks {
182
+ parser.parse_into(chunk, &mut output).unwrap();
183
+ }
184
+ output.append(parser.finish().unwrap());
185
+ output
186
+ }
187
+
188
+ struct PreserveReasoningParser;
189
+
190
+ impl ReasoningParser for PreserveReasoningParser {
191
+ fn create(
192
+ _tokenizer: vllm_tokenizer::DynTokenizer,
193
+ ) -> crate::reasoning::Result<Box<dyn ReasoningParser>>
194
+ where
195
+ Self: Sized + 'static,
196
+ {
197
+ Ok(Box::new(Self))
198
+ }
199
+
200
+ fn preserve_special_tokens(&self) -> bool {
201
+ true
202
+ }
203
+
204
+ fn push(&mut self, delta: &str) -> crate::reasoning::Result<ReasoningDelta> {
205
+ Ok(ReasoningDelta {
206
+ reasoning: None,
207
+ content: Some(delta.to_string()),
208
+ })
209
+ }
210
+ }
211
+
212
+ struct PreserveToolParser;
213
+
214
+ impl ToolParser for PreserveToolParser {
215
+ fn create(_tools: &[Tool]) -> crate::tool::Result<Box<dyn ToolParser>>
216
+ where
217
+ Self: Sized + 'static,
218
+ {
219
+ Ok(Box::new(Self))
220
+ }
221
+
222
+ fn preserve_special_tokens(&self) -> bool {
223
+ true
224
+ }
225
+
226
+ fn parse_into(
227
+ &mut self,
228
+ chunk: &str,
229
+ output: &mut crate::tool::ToolParserOutput,
230
+ ) -> crate::tool::Result<()> {
231
+ output.normal_text.push_str(chunk);
232
+ Ok(())
233
+ }
234
+
235
+ fn finish(&mut self) -> crate::tool::Result<crate::tool::ToolParserOutput> {
236
+ Ok(crate::tool::ToolParserOutput::default())
237
+ }
238
+
239
+ fn reset(&mut self) -> String {
240
+ String::new()
241
+ }
242
+ }
243
+
244
+ struct PartialThenErrorToolParser;
245
+
246
+ impl ToolParser for PartialThenErrorToolParser {
247
+ fn create(_tools: &[Tool]) -> crate::tool::Result<Box<dyn ToolParser>>
248
+ where
249
+ Self: Sized + 'static,
250
+ {
251
+ Ok(Box::new(Self))
252
+ }
253
+
254
+ fn parse_into(
255
+ &mut self,
256
+ _chunk: &str,
257
+ output: &mut crate::tool::ToolParserOutput,
258
+ ) -> crate::tool::Result<()> {
259
+ output.normal_text.push_str("committed");
260
+ Err(crate::tool::ToolParserError::ParsingFailed {
261
+ message: "synthetic failure".to_string(),
262
+ })
263
+ }
264
+
265
+ fn finish(&mut self) -> crate::tool::Result<crate::tool::ToolParserOutput> {
266
+ Ok(crate::tool::ToolParserOutput::default())
267
+ }
268
+
269
+ fn reset(&mut self) -> String {
270
+ String::new()
271
+ }
272
+ }
273
+
274
+ #[test]
275
+ fn combined_parser_emits_reasoning_and_text() {
276
+ let tokenizer = Arc::new(FakeTokenizer);
277
+ let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
278
+ let mut parser = CombinedParser::new(Some(reasoning), None);
279
+
280
+ let output = collect(&mut parser, &["<think>work</think>answer"]);
281
+
282
+ assert_eq!(
283
+ output.events,
284
+ vec![
285
+ UnifiedParserEvent::Reasoning("work".to_string()),
286
+ UnifiedParserEvent::Text("answer".to_string()),
287
+ ]
288
+ );
289
+ }
290
+
291
+ #[test]
292
+ fn combined_parser_emits_tool_calls_from_visible_content() {
293
+ let tool = Qwen3XmlToolParser::create(&test_tools()).unwrap();
294
+ let mut parser = CombinedParser::new(None, Some(tool));
295
+ assert!(matches!(
296
+ parser.structural_tag_model(),
297
+ Some(crate::tool::StructuralTagModel::Qwen3)
298
+ ));
299
+
300
+ let output = collect(
301
+ &mut parser,
302
+ &[r#"<tool_call>
303
+{"name":"get_weather","arguments":{"location":"Paris"}}
304
+</tool_call>"#],
305
+ );
306
+
307
+ assert_eq!(
308
+ output.events,
309
+ vec![
310
+ UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
311
+ tool_index: 0,
312
+ name: Some("get_weather".to_string()),
313
+ arguments: String::new(),
314
+ }),
315
+ UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
316
+ tool_index: 0,
317
+ name: None,
318
+ arguments: r#"{"location":"Paris"}"#.to_string(),
319
+ }),
320
+ ]
321
+ );
322
+ }
323
+
324
+ #[test]
325
+ fn combined_parser_preserves_tool_output_on_parse_error() {
326
+ let mut parser = CombinedParser::new(None, Some(Box::new(PartialThenErrorToolParser)));
327
+ let mut output = UnifiedParserOutput::default();
328
+
329
+ let error = parser.parse_into("bad", &mut output).unwrap_err();
330
+
331
+ assert!(matches!(error, crate::unified::UnifiedParserError::Tool(_)));
332
+ assert_eq!(
333
+ output.events,
334
+ vec![UnifiedParserEvent::Text("committed".to_string())]
335
+ );
336
+ }
337
+
338
+ #[test]
339
+ fn combined_parser_preserves_special_tokens_when_either_inner_parser_needs_it() {
340
+ let mut parser = CombinedParser::new(Some(Box::new(PreserveReasoningParser)), None);
341
+ assert!(parser.preserve_special_tokens());
342
+
343
+ parser = CombinedParser::new(None, Some(Box::new(PreserveToolParser)));
344
+ assert!(parser.preserve_special_tokens());
345
+ }
346
+}
rust/src/parser/src/unified/mod.rs
new
+114
@@ -0,0 +1,114 @@
1
+//! Unified parser interface for reasoning and tool-call deltas.
2
+
3
+mod combined;
4
+
5
+use thiserror::Error;
6
+use vllm_tokenizer::DynTokenizer;
7
+
8
+pub use combined::CombinedParser;
9
+
10
+use crate::reasoning::ReasoningError;
11
+use crate::tool::{StructuralTagModel, Tool, ToolCallDelta, ToolParserError, ToolParserOutput};
12
+
13
+/// Result alias for unified parser operations.
14
+pub type Result<T> = std::result::Result<T, UnifiedParserError>;
15
+
16
+/// One parsed event emitted by a unified parser.
17
+#[derive(Debug, Clone, PartialEq, Eq)]
18
+pub enum UnifiedParserEvent {
19
+ /// Normal assistant-visible text.
20
+ Text(String),
21
+ /// Reasoning text hidden from the normal content stream.
22
+ Reasoning(String),
23
+ /// A tool-call update extracted from visible assistant text.
24
+ ToolCall(ToolCallDelta),
25
+}
26
+
27
+/// Result of advancing unified parsing with one assistant-text input.
28
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
29
+pub struct UnifiedParserOutput {
30
+ /// Ordered parser events committed by this input.
31
+ pub events: Vec<UnifiedParserEvent>,
32
+}
33
+
34
+impl UnifiedParserOutput {
35
+ /// Append one visible text event if `delta` is non-empty.
36
+ pub fn push_text(&mut self, delta: String) {
37
+ if delta.is_empty() {
38
+ return;
39
+ }
40
+ self.events.push(UnifiedParserEvent::Text(delta));
41
+ }
42
+
43
+ /// Append one reasoning text event if `delta` is non-empty.
44
+ pub fn push_reasoning(&mut self, delta: String) {
45
+ if delta.is_empty() {
46
+ return;
47
+ }
48
+ self.events.push(UnifiedParserEvent::Reasoning(delta));
49
+ }
50
+
51
+ /// Append parsed tool parser output as unified events.
52
+ pub fn append_tool_output(&mut self, output: ToolParserOutput) {
53
+ // TODO: make ToolParserOutput carry ordered events and remove this text-first flattening.
54
+ self.push_text(output.normal_text);
55
+ self.events.extend(output.calls.into_iter().map(UnifiedParserEvent::ToolCall));
56
+ }
57
+
58
+ /// Append another parser output onto this one.
59
+ pub fn append(&mut self, mut other: Self) {
60
+ self.events.append(&mut other.events);
61
+ }
62
+}
63
+
64
+/// Incremental parser that extracts reasoning and tool-call events from assistant output.
65
+pub trait UnifiedParser: Send {
66
+ /// Construct a boxed parser instance for one request stream.
67
+ fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
68
+ where
69
+ Self: Sized + 'static;
70
+
71
+ /// Initialize parser state from prompt token IDs before output deltas arrive.
72
+ fn initialize(&mut self, _prompt_token_ids: &[u32]) -> Result<()> {
73
+ Ok(())
74
+ }
75
+
76
+ /// Return whether decoded output must preserve tokenizer special tokens.
77
+ fn preserve_special_tokens(&self) -> bool {
78
+ false
79
+ }
80
+
81
+ /// Return the xgrammar structural-tag model used for strict tool calling.
82
+ fn structural_tag_model(&self) -> Option<StructuralTagModel> {
83
+ None
84
+ }
85
+
86
+ /// Return the parser-provided ID for a tool call by index, if the model emitted one.
87
+ fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
88
+ None
89
+ }
90
+
91
+ /// Feed one decoded text delta into the parser, appending committed output into `output`.
92
+ fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()>;
93
+
94
+ /// Flush any buffered parser state at end of stream.
95
+ fn finish(&mut self) -> Result<UnifiedParserOutput> {
96
+ Ok(UnifiedParserOutput::default())
97
+ }
98
+
99
+ /// Clear parser state and return currently uncommitted buffered text.
100
+ fn reset(&mut self) -> String {
101
+ String::new()
102
+ }
103
+}
104
+
105
+/// Errors produced while creating or running unified parsers.
106
+#[derive(Debug, Error)]
107
+pub enum UnifiedParserError {
108
+ #[error("combined parser is constructed from split parser instances")]
109
+ CombinedParserConstructor,
110
+ #[error(transparent)]
111
+ Reasoning(#[from] ReasoningError),
112
+ #[error(transparent)]
113
+ Tool(#[from] ToolParserError),
114
+}
rust/src/reasoning-parser/Cargo.toml
deleted
-12
@@ -1,12 +0,0 @@
1
-[package]
2
-name = "vllm-reasoning-parser"
3
-version.workspace = true
4
-edition.workspace = true
5
-license.workspace = true
6
-
7
-[dependencies]
8
-thiserror.workspace = true
9
-vllm-tokenizer.workspace = true
10
-
11
-[lints]
12
-workspace = true
tools/build_rust.py
+1
-1
@@ -27,7 +27,7 @@ def rust_extensions(*, optional: bool = False) -> list[RustExtension]:
27
),
28
RustExtension(
29
target="vllm._rust_tool_parser",
30
- path="rust/src/tool-parser/python/Cargo.toml",
30
+ path="rust/src/parser/python/Cargo.toml",
31
features=["pyo3/abi3-py38"],
32
binding=Binding.PyO3,
33
optional=optional,