@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ceade1952c

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 01:00 UTC

[BugFix] Support custom tool parsers when tool_choice is `required` and named function (#39870)

Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>

Jared Wen committed Apr 18, 2026 at 00:38 UTC ceade1952c7aebfbf328daa7dd40877c5cd8e8a5

5 files changed +100 -12

vllm/entrypoints/openai/chat_completion/serving.py

+39 -6

@@ -557,6 +557,20 @@ class OpenAIServingChat(OpenAIServing):

557 and self._should_stream_with_auto_tool_parsing(request)

558 )

559

560 + # Determine whether required/named tool_choice should fall back to

561 + # the auto tool_parser path instead of the standard JSON-based parsing.

562 + # This happens when the parser declares supports_required_and_named=False

563 + # (e.g. GLM models that output XML instead of JSON).

564 + tool_choice_uses_parser = (

565 + self.tool_parser is not None

566 + and not self.tool_parser.supports_required_and_named

567 + and request.tools

568 + and (

569 + request.tool_choice == "required"

570 + or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)

571 + )

572 + )

573 +

574 all_previous_token_ids: list[list[int]] | None

575 function_name_returned = [False] * num_choices

576 if self.tool_call_id_type == "kimi_k2":

@@ -569,7 +583,12 @@ class OpenAIServingChat(OpenAIServing):

583

584 # Only one of these will be used, thus previous_texts and

585 # all_previous_token_ids will not be used twice in the same iteration.

572 - if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:

586 + if (

587 + is_mistral_grammar_path

588 + or tool_choice_auto

589 + or tool_choice_uses_parser

590 + or reasoning_parser

591 + ):

592 # These are only required in "auto" tool choice case

593 all_previous_token_ids = [[] for _ in range(num_choices)]

594 reasoning_end_arr = [False] * num_choices

@@ -764,7 +783,12 @@ class OpenAIServingChat(OpenAIServing):

783 delta_message: DeltaMessage | None

784

785 # just update previous_texts and previous_token_ids

767 - if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:

786 + if (

787 + is_mistral_grammar_path

788 + or tool_choice_auto

789 + or tool_choice_uses_parser

790 + or reasoning_parser

791 + ):

792 assert previous_texts is not None

793 assert all_previous_token_ids is not None

794 previous_text = previous_texts[i]

@@ -813,7 +837,9 @@ class OpenAIServingChat(OpenAIServing):

837 if result.tools_called:

838 tools_streamed[i] = True

839 # handle streaming deltas for tools with named tool_choice

816 - elif tool_choice_function_name:

840 + # Skip when tool_choice_uses_parser so it falls through

841 + # to the auto tool_parser branches below.

842 + elif tool_choice_function_name and not tool_choice_uses_parser:

843 # When encountering think end id in prompt_token_ids

844 # i.e {"enable_thinking": False},

845 # check BEFORE calling the parser to avoid a spurious

@@ -851,7 +877,6 @@ class OpenAIServingChat(OpenAIServing):

877 ):

878 reasoning_end_arr[i] = True

879 if delta_message and delta_message.content:

854 - # This need to be added to next `delta_text`

880 current_text = delta_message.content

881 delta_message.content = None

882 else:

@@ -896,7 +921,12 @@ class OpenAIServingChat(OpenAIServing):

921 )

922 tools_streamed[i] = True

923

899 - elif request.tool_choice == "required":

924 + # Skip when tool_choice_uses_parser so it falls through

925 + # to the auto tool_parser branches below.

926 + elif (

927 + request.tool_choice == "required"

928 + and not tool_choice_uses_parser

929 + ):

930 assert previous_texts is not None

931 previous_text = previous_texts[i]

932 current_text = previous_text + delta_text

@@ -966,7 +996,10 @@ class OpenAIServingChat(OpenAIServing):

996

997 # update the previous values for the next iteration

998 if (

969 - is_mistral_grammar_path or tool_choice_auto or reasoning_parser

999 + is_mistral_grammar_path

1000 + or tool_choice_auto

1001 + or tool_choice_uses_parser

1002 + or reasoning_parser

1003 ) and not self.use_harmony:

1004 assert previous_texts is not None

1005 assert all_previous_token_ids is not None

vllm/entrypoints/openai/engine/serving.py

+26 -5

@@ -627,7 +627,7 @@ class OpenAIServing:

627 and isinstance(request.tool_choice, ToolChoiceFunction)

628 ):

629 assert content is not None

630 - # Forced Function Call

630 + # Forced Function Call (Responses API)

631 function_calls.append(

632 FunctionCall(name=request.tool_choice.name, arguments=content)

633 )

@@ -636,14 +636,20 @@ class OpenAIServing:

636 not use_mistral_tool_parser

637 and request.tool_choice

638 and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)

639 + and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)

640 ):

641 + # Named function with standard JSON-based parsing

642 assert content is not None

641 - # Forced Function Call

643 function_calls.append(

644 FunctionCall(name=request.tool_choice.function.name, arguments=content)

645 )

646 content = None # Clear content since tool is called.

646 - elif not use_mistral_tool_parser and request.tool_choice == "required":

647 + elif (

648 + not use_mistral_tool_parser

649 + and request.tool_choice == "required"

650 + and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)

651 + ):

652 + # "required" with standard JSON-based parsing

653 tool_calls = []

654 with contextlib.suppress(ValidationError):

655 content = content or ""

@@ -662,15 +668,30 @@ class OpenAIServing:

668 use_mistral_tool_parser

669 or (

670 enable_auto_tools

665 - and (request.tool_choice == "auto" or request.tool_choice is None)

671 + and (

672 + request.tool_choice == "auto"

673 + or request.tool_choice is None

674 + or (

675 + not tool_parser_cls.supports_required_and_named

676 + and request.tools

677 + and (

678 + request.tool_choice == "required"

679 + or isinstance(

680 + request.tool_choice,

681 + ChatCompletionNamedToolChoiceParam,

682 + )

683 + )

684 + )

685 + )

686 )

687 ):

688 + # Automatic Tool Call Parsing (also used as fallback for

689 + # required/named when supports_required_and_named=False)

690 if tokenizer is None:

691 raise ValueError(

692 "Tokenizer not available when `skip_tokenizer_init=True`"

693 )

694

673 - # Automatic Tool Call Parsing

695 try:

696 tool_parser = tool_parser_cls(tokenizer, request.tools)

697 except RuntimeError as e:

vllm/tool_parsers/abstract_tool_parser.py

+11

@@ -44,6 +44,17 @@ class ToolParser:

44 derived classes.

45 """

47 + # When True (default), the serving layer uses the standard JSON-based

48 + # parsing for tool_choice="required" and named function tool_choice,

49 + # which works for models where guided decoding produces well-formed

50 + # JSON output (e.g. Hermes).

51 + # Subclasses set False when the standard parsing does not work for

52 + # their model's output format (e.g. GLM models that use XML). When

53 + # False, the serving layer falls back to the tool_parser's

54 + # extract_tool_calls / extract_tool_calls_streaming methods for

55 + # required/named tool_choice, treating them the same as "auto".

56 + supports_required_and_named: bool = True

57 +

58 def __init__(

59 self,

60 tokenizer: TokenizerLike,

vllm/tool_parsers/glm47_moe_tool_parser.py

@@ -23,6 +23,8 @@ logger = init_logger(__name__)

25 class Glm47MoeModelToolParser(Glm4MoeModelToolParser):

26 + supports_required_and_named = False

27 +

28 def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):

29 super().__init__(tokenizer, tools)

30 # GLM-4.7 format: <tool_call>func_name[<arg_key>...]*</tool_call>

vllm/tool_parsers/glm4_moe_tool_parser.py

+22 -1

@@ -20,6 +20,7 @@ import regex as re

21 from vllm.entrypoints.chat_utils import make_tool_call_id

22 from vllm.entrypoints.openai.chat_completion.protocol import (

23 + ChatCompletionNamedToolChoiceParam,

24 ChatCompletionRequest,

25 )

26 from vllm.entrypoints.openai.engine.protocol import (

@@ -50,6 +51,8 @@ class Glm4MoeModelToolParser(ToolParser):

51 call, and diffs against what was previously sent to emit only new content.

52 """

54 + supports_required_and_named = False

55 +

56 def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):

57 super().__init__(tokenizer, tools)

58 # Stateful streaming fields

@@ -156,7 +159,25 @@ class Glm4MoeModelToolParser(ToolParser):

159 def adjust_request(

160 self, request: ChatCompletionRequest | ResponsesRequest

161 ) -> ChatCompletionRequest | ResponsesRequest:

159 - """Adjust request parameters for tool call token handling."""

162 + """Adjust request parameters for tool call token handling.

163 +

164 + For required/named tool_choice, skip setting structured_outputs

165 + because GLM models output tool calls in XML format (per chat

166 + template). Guided decoding would force JSON output, conflicting

167 + with the XML format and causing parsing failures.

168 + """

169 + if request.tools:

170 + tc = request.tool_choice

171 + if tc == "required" or isinstance(tc, ChatCompletionNamedToolChoiceParam):

172 + # Do NOT call super().adjust_request() for required/named,

173 + # because it would set structured_outputs and force JSON

174 + # output via guided decoding. GLM models use XML tool-call

175 + # syntax (defined in the chat template), so guided decoding

176 + # must be skipped to let the model output XML freely.

177 + # The tool_parser handles extraction from XML output.

178 + if request.tool_choice != "none":

179 + request.skip_special_tokens = False

180 + return request

181 request = super().adjust_request(request)

182 if request.tools and request.tool_choice != "none":

183 # Ensure tool call tokens (<tool_call>, </tool_call>) are not skipped