@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 40eac9a9d9

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Rust Frontend] Support `parallel_tool_calls = false` (#44760)

Signed-off-by: zhoujinyu <2319109590@qq.com>

FAUST committed Jun 15, 2026 at 15:50 UTC 40eac9a9d92bba51ad49ca777a5517ee212ea394

8 files changed +135 -23

rust/src/chat/src/output/default/mod.rs

+4 -1

@@ -37,6 +37,7 @@ trait_set! {

37 pub struct DefaultChatOutputProcessor {

38 reasoning_parser: Option<Box<dyn ReasoningParser>>,

39 tool_parser: Option<Box<dyn ToolParser>>,

40 + parallel_tool_calls: bool,

41 }

43 impl DefaultChatOutputProcessor {

@@ -74,6 +75,7 @@ impl DefaultChatOutputProcessor {

75 Ok(Self {

76 reasoning_parser,

77 tool_parser,

78 + parallel_tool_calls: request.parallel_tool_calls,

79 })

80 }

@@ -86,6 +88,7 @@ impl DefaultChatOutputProcessor {

88 Self {

89 reasoning_parser: None,

90 tool_parser: None,

91 + parallel_tool_calls: true,

92 }

93 }

@@ -159,7 +162,7 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {

162 fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {

163 let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);

164 let tool = tool_event_stream(reasoning, self.tool_parser);

162 - let structured = structured_chat_event_stream(tool);

165 + let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);

166

167 Ok(structured.boxed())

168 }

rust/src/chat/src/output/default/tool.rs

+9 -7

@@ -473,7 +473,7 @@ mod tests {

473 })));

474 let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();

475 let assistant_events = tool_event_stream(stream::iter(events), Some(parser));

476 - let chat_events = structured_chat_event_stream(assistant_events);

476 + let chat_events = structured_chat_event_stream(assistant_events, true);

477

478 ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))

479 .collect_message()

@@ -717,9 +717,10 @@ mod tests {

717

718 let message = ChatEventStream::new(

719 "req_fallback".to_string(),

720 - Box::pin(structured_chat_event_stream(stream::iter(

721 - events.into_iter().map(Ok),

722 - ))),

720 + Box::pin(structured_chat_event_stream(

721 + stream::iter(events.into_iter().map(Ok)),

722 + true,

723 + )),

724 )

725 .collect_message()

726 .await

@@ -968,9 +969,10 @@ mod tests {

969 ));

970 let collected = ChatEventStream::new(

971 "req_final_only".to_string(),

971 - Box::pin(structured_chat_event_stream(stream::iter(

972 - events.into_iter().map(Ok),

973 - ))),

972 + Box::pin(structured_chat_event_stream(

973 + stream::iter(events.into_iter().map(Ok)),

974 + true,

975 + )),

976 )

977 .collect_message()

978 .await

rust/src/chat/src/output/harmony/mod.rs

+7 -1

@@ -35,6 +35,7 @@ use crate::request::ChatRequest;

35 pub struct HarmonyChatOutputProcessor {

36 encoding: &'static HarmonyEncoding,

37 tool_calls_enabled: bool,

38 + parallel_tool_calls: bool,

39 }

41 #[derive(Debug, Clone, PartialEq, Eq, Hash)]

@@ -76,6 +77,7 @@ impl HarmonyChatOutputProcessor {

77 Ok(Self {

78 encoding: harmony_encoding()?,

79 tool_calls_enabled: request.tool_parsing_enabled(),

80 + parallel_tool_calls: request.parallel_tool_calls,

81 })

82 }

83 }

@@ -110,7 +112,11 @@ impl ChatOutputProcessor for HarmonyChatOutputProcessor {

112 fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {

113 let assistant =

114 harmony_assistant_event_stream(decoded, self.encoding, self.tool_calls_enabled);

113 - Ok(crate::output::structured::structured_chat_event_stream(assistant).boxed())

115 + Ok(crate::output::structured::structured_chat_event_stream(

116 + assistant,

117 + self.parallel_tool_calls,

118 + )

119 + .boxed())

120 }

121 }

122

rust/src/chat/src/output/structured.rs

+81 -7

@@ -53,16 +53,22 @@ struct StructuredEventState {

53 open_tool_call: Option<OpenToolCall>,

54 /// Next OpenAI-compatible tool-call ordinal.

55 next_tool_call_index: usize,

56 + /// Whether more than one tool call may be surfaced northbound.

57 + parallel_tool_calls: bool,

58 + /// Whether the current tool-call parse is being suppressed.

59 + suppressing_tool_call: bool,

60 }

62 impl StructuredEventState {

63 /// Create one fresh assembly state for a new streamed response.

60 - fn new() -> Self {

64 + fn new(parallel_tool_calls: bool) -> Self {

65 Self {

66 message: AssistantMessage::default(),

67 open_text_block: None,

68 open_tool_call: None,

69 next_tool_call_index: 0,

70 + parallel_tool_calls,

71 + suppressing_tool_call: false,

72 }

73 }

@@ -98,6 +104,12 @@ impl StructuredEventState {

104

105 let index = self.next_tool_call_index;

106 self.next_tool_call_index += 1;

107 + if !self.parallel_tool_calls && index >= 1 {

108 + self.suppressing_tool_call = true;

109 + return Ok(events);

110 + }

111 +

112 + self.suppressing_tool_call = false;

113 self.open_tool_call = Some(OpenToolCall {

114 index,

115 id: id.clone(),

@@ -110,6 +122,10 @@ impl StructuredEventState {

122

123 /// Append one incremental tool-call arguments delta.

124 fn push_tool_call_arguments(&mut self, delta: String) -> Result<Vec<ChatEvent>> {

125 + if self.suppressing_tool_call {

126 + return Ok(Vec::new());

127 + }

128 +

129 let mut events = Vec::new();

130 let Some(open_tool_call) = self.open_tool_call.as_mut() else {

131 return Err(Error::ToolCallStreamInvariant {

@@ -207,6 +223,11 @@ impl StructuredEventState {

223

224 /// Finalize the currently open tool call, if present.

225 fn close_open_tool_call(&mut self, events: &mut Vec<ChatEvent>) {

226 + if self.suppressing_tool_call {

227 + self.suppressing_tool_call = false;

228 + return;

229 + }

230 +

231 let Some(open_tool_call) = self.open_tool_call.take() else {

232 return;

233 };

@@ -229,11 +250,12 @@ impl StructuredEventState {

250 #[try_stream]

251 pub(crate) async fn structured_chat_event_stream(

252 stream: impl AssistantEventStream,

253 + parallel_tool_calls: bool,

254 mut y: TryYielder<ChatEvent, Error>,

255 ) -> Result<()> {

256 pin_mut!(stream);

257

236 - let mut state = StructuredEventState::new();

258 + let mut state = StructuredEventState::new(parallel_tool_calls);

259

260 while let Some(event) = stream.next().await.transpose()? {

261 match event {

@@ -315,7 +337,7 @@ mod tests {

337 }),

338 ]);

339

318 - let events = structured_chat_event_stream(events)

340 + let events = structured_chat_event_stream(events, true)

341 .collect::<Vec<_>>()

342 .await

343 .into_iter()

@@ -369,7 +391,7 @@ mod tests {

391 }),

392 ]);

393

372 - let events = structured_chat_event_stream(events)

394 + let events = structured_chat_event_stream(events, true)

395 .collect::<Vec<_>>()

396 .await

397 .into_iter()

@@ -420,7 +442,7 @@ mod tests {

442 }),

443 ]);

444

423 - let events = structured_chat_event_stream(events)

445 + let events = structured_chat_event_stream(events, true)

446 .collect::<Vec<_>>()

447 .await

448 .into_iter()

@@ -471,7 +493,7 @@ mod tests {

493 }),

494 ]);

495

474 - let events = structured_chat_event_stream(events)

496 + let events = structured_chat_event_stream(events, true)

497 .collect::<Vec<_>>()

498 .await

499 .into_iter()

@@ -499,7 +521,7 @@ mod tests {

521 delta: "{}".to_string(),

522 })]);

523

502 - let err = structured_chat_event_stream(events)

524 + let err = structured_chat_event_stream(events, true)

525 .collect::<Vec<_>>()

526 .await

527 .into_iter()

@@ -509,4 +531,56 @@ mod tests {

531

532 assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));

533 }

534 +

535 + #[tokio::test]

536 + async fn structured_stream_suppresses_later_tool_calls_when_parallel_disabled() {

537 + let events = stream::iter(vec![

538 + Ok(AssistantEvent::ToolCallStart {

539 + id: "call_1".to_string(),

540 + name: "first".to_string(),

541 + }),

542 + Ok(AssistantEvent::ToolCallArgumentsDelta {

543 + delta: r#"{"a":1}"#.to_string(),

544 + }),

545 + Ok(AssistantEvent::ToolCallStart {

546 + id: "call_2".to_string(),

547 + name: "second".to_string(),

548 + }),

549 + Ok(AssistantEvent::ToolCallArgumentsDelta {

550 + delta: r#"{"b":2}"#.to_string(),

551 + }),

552 + Ok(AssistantEvent::Done {

553 + usage: vllm_llm::TokenUsage {

554 + prompt_token_count: 1,

555 + output_token_count: 1,

556 + cached_token_count: 0,

557 + },

558 + finish_reason: FinishReason::stop_eos(),

559 + kv_transfer_params: None,

560 + }),

561 + ]);

562 +

563 + let events = structured_chat_event_stream(events, false)

564 + .collect::<Vec<_>>()

565 + .await

566 + .into_iter()

567 + .collect::<crate::Result<Vec<_>>>()

568 + .unwrap();

569 +

570 + assert!(matches!(

571 + events[0],

572 + ChatEvent::ToolCallStart { index: 0, .. }

573 + ));

574 + assert!(matches!(

575 + events[1],

576 + ChatEvent::ToolCallArgumentsDelta { index: 0, .. }

577 + ));

578 + assert!(matches!(events[2], ChatEvent::ToolCallEnd { index: 0, .. }));

579 + let ChatEvent::Done { message, .. } = &events[3] else {

580 + panic!("expected done");

581 + };

582 + let tool_calls = message.tool_calls().collect::<Vec<_>>();

583 + assert_eq!(tool_calls.len(), 1);

584 + assert_eq!(tool_calls[0].name, "first");

585 + }

586 }

rust/src/chat/src/request.rs

@@ -406,6 +406,10 @@ pub struct ChatRequest {

406 pub tools: Vec<ChatTool>,

407 /// Tool-choice behavior for this request.

408 pub tool_choice: ChatToolChoice,

409 + /// Whether the model may return more than one tool call per response.

410 + ///

411 + /// When `false`, only the first parsed tool call is surfaced northbound.

412 + pub parallel_tool_calls: bool,

413 /// Text decode options for incremental detokenization.

414 pub decode_options: TextDecodeOptions,

415 /// Whether to emit intermediate northbound content deltas before the

@@ -442,6 +446,7 @@ impl ChatRequest {

446 chat_options: ChatOptions::default(),

447 tools: Vec::new(),

448 tool_choice: ChatToolChoice::None,

449 + parallel_tool_calls: true,

450 decode_options: TextDecodeOptions::default(),

451 intermediate: true,

452 priority: 0,

rust/src/server/src/routes/openai/chat_completions/convert.rs

+28

@@ -142,6 +142,7 @@ pub(super) fn prepare_chat_request(

142 },

143 tools: convert_tools(request.tools)?,

144 tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,

145 + parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),

146 decode_options: vllm_text::output::TextDecodeOptions {

147 skip_special_tokens: request.skip_special_tokens,

148 include_stop_str_in_output: request.include_stop_str_in_output,

@@ -412,6 +413,33 @@ mod tests {

413 }

414 }

415

416 + #[test]

417 + fn prepare_chat_request_maps_parallel_tool_calls() {

418 + let mut request = base_request();

419 + request.parallel_tool_calls = Some(false);

420 +

421 + let prepared = prepare_chat_request(

422 + request,

423 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

424 + ResolvedRequestContext::default(),

425 + )

426 + .expect("request is valid");

427 +

428 + assert!(!prepared.chat_request.parallel_tool_calls);

429 + }

430 +

431 + #[test]

432 + fn prepare_chat_request_defaults_parallel_tool_calls_to_true() {

433 + let prepared = prepare_chat_request(

434 + base_request(),

435 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),

436 + ResolvedRequestContext::default(),

437 + )

438 + .expect("request is valid");

439 +

440 + assert!(prepared.chat_request.parallel_tool_calls);

441 + }

442 +

443 #[test]

444 fn prepare_chat_request_maps_text_parts() {

445 let mut request = base_request();

rust/src/server/src/routes/openai/chat_completions/validate.rs

-7

@@ -93,13 +93,6 @@ pub(super) fn validate_request_compat(

93 // ---- Reject parameters that are accepted for deserialization but not yet

94 // implemented ----

96 - if request.parallel_tool_calls.is_some() {

97 - bail_invalid_request!(

98 - param = "parallel_tool_calls",

99 - "parallel_tool_calls is not supported."

100 - );

101 - }

102 -

96 reject_non_default(

97 request.length_penalty.as_ref(),

98 "length_penalty",

rust/src/server/src/routes/tokenize/types.rs

@@ -83,6 +83,7 @@ impl TokenizeChatRequest {

83 },

84 tools: convert_tools(self.tools)?,

85 tool_choice: ChatToolChoice::Auto,

86 + parallel_tool_calls: true,

87 decode_options: TextDecodeOptions::default(),

88 intermediate: false,

89 priority: 0,