@tduncan / Nvidia-Optimization-Stack-VLLM / commits / dc818c198d

[GPT-OSS] Strict tool call and constrained decoding for Harmony (#45560)

Signed-off-by: Yifan Zong <yzong@redhat.com>

yzong-rh committed Aug 1, 2026 at 12:10 UTC dc818c198d3ff50a16f38eba567da006478239c8
10 files changed +632 -511
tests/entrypoints/openai/responses/test_harmony.py
+33 -23
@@ -13,7 +13,7 @@ from typing import Any
13 import pytest
14 import pytest_asyncio
15 import requests
16 -from openai import InternalServerError, NotFoundError, OpenAI
16 +from openai import NotFoundError, OpenAI
17 from openai_harmony import Message
18
19 from tests.utils import RemoteOpenAIServer
@@ -368,8 +368,12 @@ async def test_streaming_types(
368
369 @pytest.mark.asyncio
370 @pytest.mark.parametrize("model_name", [MODEL_NAME])
371 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
372 async def test_function_calling_with_streaming_types(
372 - pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str
373 + pairs_of_event_types: dict[str, str],
374 + client: OpenAI,
375 + model_name: str,
376 + tool_choice: str,
377 ):
378 """Streaming event nesting for function-calling responses."""
379
@@ -382,6 +386,7 @@ async def test_function_calling_with_streaming_types(
386 validate_events=_has_function_events,
387 input=[{"role": "user", "content": "What's the weather like in Paris today?"}],
388 tools=[GET_WEATHER_SCHEMA],
389 + tool_choice=tool_choice,
390 temperature=0.0,
391 )
392
@@ -558,7 +563,8 @@ async def test_reasoning_item(client: OpenAI, model_name: str):
563
564 @pytest.mark.asyncio
565 @pytest.mark.parametrize("model_name", [MODEL_NAME])
561 -async def test_function_calling(client: OpenAI, model_name: str):
566 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
567 +async def test_function_calling(client: OpenAI, model_name: str, tool_choice: str):
568 tools = [GET_WEATHER_SCHEMA]
569
570 response = await retry_for_tool_call(
@@ -567,8 +573,9 @@ async def test_function_calling(client: OpenAI, model_name: str):
573 expected_tool_type="function_call",
574 input="What's the weather like in Paris today?",
575 tools=tools,
576 + tool_choice=tool_choice,
577 temperature=0.0,
571 - extra_body={"request_id": "test_function_calling_non_resp"},
578 + extra_body={"request_id": f"test_function_calling_non_resp_{tool_choice}"},
579 )
580 assert response.status == "completed"
581 assert has_output_type(response, "function_call"), (
@@ -610,7 +617,10 @@ async def test_function_calling(client: OpenAI, model_name: str):
617
618 @pytest.mark.asyncio
619 @pytest.mark.parametrize("model_name", [MODEL_NAME])
613 -async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
620 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
621 +async def test_function_calling_multi_turn(
622 + client: OpenAI, model_name: str, tool_choice: str
623 +):
624 """Multi-tool, multi-turn function calling with retry at API level."""
625 tools = [
626 {
@@ -635,6 +645,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
645 expected_tool_type="function_call",
646 input="Help me plan a trip to a random place. And tell me the weather there.",
647 tools=tools,
648 + tool_choice=tool_choice,
649 temperature=0.0,
650 )
651 assert response.status == "completed"
@@ -659,6 +670,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
670 }
671 ],
672 tools=tools,
673 + tool_choice=tool_choice,
674 previous_response_id=response.id,
675 temperature=0.0,
676 )
@@ -695,20 +707,6 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
707 )
708
709
698 -@pytest.mark.asyncio
699 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
700 -async def test_function_calling_required(client: OpenAI, model_name: str):
701 - tools = [GET_WEATHER_SCHEMA]
702 -
703 - with pytest.raises(InternalServerError):
704 - await client.responses.create(
705 - model=model_name,
706 - input="What's the weather like in Paris today?",
707 - tools=tools,
708 - tool_choice="required",
709 - )
710 -
711 -
710 @pytest.mark.asyncio
711 @pytest.mark.parametrize("model_name", [MODEL_NAME])
712 async def test_system_message_with_tools(client: OpenAI, model_name: str):
@@ -726,7 +724,10 @@ async def test_system_message_with_tools(client: OpenAI, model_name: str):
724
725 @pytest.mark.asyncio
726 @pytest.mark.parametrize("model_name", [MODEL_NAME])
729 -async def test_function_calling_full_history(client: OpenAI, model_name: str):
727 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
728 +async def test_function_calling_full_history(
729 + client: OpenAI, model_name: str, tool_choice: str
730 +):
731 tools = [GET_WEATHER_SCHEMA]
732
733 input_messages = [
@@ -739,6 +740,7 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
740 expected_tool_type="function_call",
741 input=input_messages,
742 tools=tools,
743 + tool_choice=tool_choice,
744 temperature=0.0,
745 )
746 assert response.status == "completed"
@@ -772,7 +774,10 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str):
774
775 @pytest.mark.asyncio
776 @pytest.mark.parametrize("model_name", [MODEL_NAME])
775 -async def test_function_calling_with_stream(client: OpenAI, model_name: str):
777 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
778 +async def test_function_calling_with_stream(
779 + client: OpenAI, model_name: str, tool_choice: str
780 +):
781 """Function calling via streaming, with retry for non-determinism."""
782 tools = [GET_WEATHER_SCHEMA]
783 input_list = [
@@ -792,6 +797,7 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
797 validate_events=_has_function_call,
798 input=input_list,
799 tools=tools,
800 + tool_choice=tool_choice,
801 temperature=0.0,
802 )
803
@@ -853,8 +859,9 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
859
860 @pytest.mark.asyncio
861 @pytest.mark.parametrize("model_name", [MODEL_NAME])
862 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
863 async def test_function_calling_no_code_interpreter_events(
857 - client: OpenAI, model_name: str
864 + client: OpenAI, model_name: str, tool_choice: str
865 ):
866 """Verify that function calls don't trigger code_interpreter events.
867
@@ -880,6 +887,7 @@ async def test_function_calling_no_code_interpreter_events(
887 validate_events=_has_function_call,
888 input=input_list,
889 tools=tools,
890 + tool_choice=tool_choice,
891 temperature=0.0,
892 )
893
@@ -1048,8 +1056,9 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server):
1056
1057 @pytest.mark.asyncio
1058 @pytest.mark.parametrize("model_name", [MODEL_NAME])
1059 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
1060 async def test_function_call_with_previous_input_messages(
1052 - client: OpenAI, model_name: str
1061 + client: OpenAI, model_name: str, tool_choice: str
1062 ):
1063 """Multi-turn function calling using previous_input_messages."""
1064 tools = [
@@ -1074,6 +1083,7 @@ async def test_function_call_with_previous_input_messages(
1083 expected_tool_type="function_call",
1084 input="What is the horoscope for Aquarius today?",
1085 tools=tools,
1086 + tool_choice=tool_choice,
1087 temperature=0.0,
1088 extra_body={"enable_response_messages": True},
1089 max_output_tokens=1000,
tests/parser/test_harmony.py
+337
@@ -3,6 +3,7 @@
3
4 import json
5 from collections.abc import Sequence
6 +from typing import Any, Literal
7
8 import pytest
9 from openai_harmony import (
@@ -12,14 +13,18 @@ from openai_harmony import (
13 Role,
14 )
15 from transformers import AutoTokenizer
16 +from xgrammar import Grammar
17 +from xgrammar.testing import _is_grammar_accept_string
18
19 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
20 from vllm.entrypoints.openai.engine.protocol import FunctionCall
21 from vllm.entrypoints.openai.parser.harmony_utils import (
22 get_encoding,
23 )
24 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
25 from vllm.parser.harmony import HarmonyParser
26 from vllm.parser.parser_manager import ParserManager
27 +from vllm.sampling_params import StructuredOutputsParams
28
29 REASONING_MODEL_NAME = "openai/gpt-oss-20b"
30
@@ -857,3 +862,335 @@ class TestProcessChunk:
862 ("analysis", "One"),
863 ("final", "Two"),
864 ]
865 +
866 +
867 +class TestAdjustRequest:
868 + REQUEST_TEXT = "Hello"
869 + TOOL_TYPE = "function"
870 + TOOL_1_NAME = "get_user_location"
871 + TOOL_2_NAME = "get_weather"
872 + TOOLS = [
873 + {
874 + "name": TOOL_1_NAME,
875 + "parameters": {
876 + "type": "object",
877 + "properties": {},
878 + "required": [],
879 + },
880 + },
881 + {
882 + "name": TOOL_2_NAME,
883 + "parameters": {
884 + "type": "object",
885 + "properties": {"city": {"type": "string"}},
886 + "required": ["city"],
887 + },
888 + },
889 + ]
890 + OUTPUT_SCHEMA = {
891 + "type": "object",
892 + "properties": {"answer": {"type": "string"}},
893 + "required": ["answer"],
894 + }
895 +
896 + ANALYSIS = "<|channel|>analysis<|message|>analysis message<|end|><|start|>assistant"
897 + COMMENTARY = (
898 + "<|channel|>commentary<|message|>commentary message<|end|><|start|>assistant"
899 + )
900 + TOOL_CALL_1 = (
901 + ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_1_NAME} json<|message|>"
902 + "{}<|call|>"
903 + )
904 + TOOL_CALL_2 = (
905 + ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_2_NAME} json<|message|>"
906 + '{"city": "Tokyo"}<|call|>'
907 + )
908 + FINAL_JSON_SCHEMA = (
909 + ANALYSIS
910 + + '<|channel|>final <|constrain|>json<|message|>{"answer": "Tokyo"}<|end|>'
911 + )
912 + FINAL_JSON_OBJECT = (
913 + ANALYSIS
914 + + '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>'
915 + )
916 + FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>any<|end|>"
917 + FINAL_REGEX = ANALYSIS + "<|channel|>final<|message|>regex<|end|>"
918 + FINAL_CHOICE = ANALYSIS + "<|channel|>final<|message|>choice1<|end|>"
919 + FINAL_GRAMMAR = ANALYSIS + "<|channel|>final<|message|>grammar<|end|>"
920 + FINAL_STRUCTURAL_TAG = ANALYSIS + "<|channel|>final<|message|>tag content<|end|>"
921 + ADMISSION_SAMPLES = (
922 + "COMMENTARY",
923 + "TOOL_CALL_1",
924 + "TOOL_CALL_2",
925 + "FINAL_JSON_SCHEMA",
926 + "FINAL_JSON_OBJECT",
927 + "FINAL_TEXT_ONLY",
928 + "FINAL_REGEX",
929 + "FINAL_CHOICE",
930 + "FINAL_GRAMMAR",
931 + "FINAL_STRUCTURAL_TAG",
932 + )
933 +
934 + @staticmethod
935 + def _build_request(
936 + request_kind: Literal["chat", "responses"],
937 + tool_choice: str = "none",
938 + strict_tools: bool = False,
939 + response_format_type: str | None = None,
940 + structured_outputs: StructuredOutputsParams | None = None,
941 + ) -> ChatCompletionRequest | ResponsesRequest:
942 + data: dict[str, Any] = {
943 + "model": REASONING_MODEL_NAME,
944 + }
945 + if request_kind == "chat":
946 + data["messages"] = [
947 + {
948 + "role": "user",
949 + "content": TestAdjustRequest.REQUEST_TEXT,
950 + }
951 + ]
952 + else:
953 + data["input"] = TestAdjustRequest.REQUEST_TEXT
954 +
955 + if request_kind == "chat":
956 + data["tools"] = [
957 + {
958 + "type": TestAdjustRequest.TOOL_TYPE,
959 + "function": {"strict": strict_tools, **tool_def},
960 + }
961 + for tool_def in TestAdjustRequest.TOOLS
962 + ]
963 + data["tool_choice"] = (
964 + {
965 + "type": TestAdjustRequest.TOOL_TYPE,
966 + "function": {"name": TestAdjustRequest.TOOL_2_NAME},
967 + }
968 + if tool_choice == "named"
969 + else tool_choice
970 + )
971 + else:
972 + data["tools"] = [
973 + {
974 + "type": TestAdjustRequest.TOOL_TYPE,
975 + "strict": strict_tools,
976 + **tool_def,
977 + }
978 + for tool_def in TestAdjustRequest.TOOLS
979 + ]
980 + data["tool_choice"] = (
981 + {
982 + "type": TestAdjustRequest.TOOL_TYPE,
983 + "name": TestAdjustRequest.TOOL_2_NAME,
984 + }
985 + if tool_choice == "named"
986 + else tool_choice
987 + )
988 +
989 + if response_format_type == "json_schema":
990 + schema_format = {
991 + "name": "answer_format",
992 + "schema": TestAdjustRequest.OUTPUT_SCHEMA,
993 + "strict": True,
994 + }
995 + if request_kind == "chat":
996 + data["response_format"] = {
997 + "type": "json_schema",
998 + "json_schema": schema_format,
999 + }
1000 + else:
1001 + data["text"] = {
1002 + "format": {
1003 + "type": "json_schema",
1004 + **schema_format,
1005 + }
1006 + }
1007 + elif response_format_type == "json_object":
1008 + if request_kind == "chat":
1009 + data["response_format"] = {"type": "json_object"}
1010 + else:
1011 + data["text"] = {"format": {"type": "json_object"}}
1012 +
1013 + if structured_outputs is not None:
1014 + data["structured_outputs"] = structured_outputs
1015 +
1016 + if request_kind == "chat":
1017 + return ChatCompletionRequest.model_validate(data)
1018 + return ResponsesRequest.model_validate(data)
1019 +
1020 + @staticmethod
1021 + def _assert_format_cleared(
1022 + adjusted_request: ChatCompletionRequest | ResponsesRequest,
1023 + ) -> None:
1024 + if isinstance(adjusted_request, ResponsesRequest):
1025 + assert adjusted_request.text is None or adjusted_request.text.format is None
1026 + else:
1027 + assert adjusted_request.response_format is None
1028 +
1029 + structured_outputs = adjusted_request.structured_outputs
1030 + assert structured_outputs is not None
1031 + assert structured_outputs.structural_tag is not None
1032 + assert structured_outputs.all_non_structural_tag_constraints_none()
1033 +
1034 + @classmethod
1035 + def _assert_structured_outputs_admission(
1036 + cls,
1037 + adjusted_request: ChatCompletionRequest | ResponsesRequest,
1038 + expected_admission: Sequence[str],
1039 + ) -> None:
1040 + structured_outputs = adjusted_request.structured_outputs
1041 + assert structured_outputs is not None
1042 + assert structured_outputs.structural_tag is not None
1043 + assert structured_outputs.all_non_structural_tag_constraints_none()
1044 +
1045 + grammar = Grammar.from_structural_tag(structured_outputs.structural_tag)
1046 + expected_admission_set = set(expected_admission)
1047 +
1048 + for sample_name in cls.ADMISSION_SAMPLES:
1049 + admitted = _is_grammar_accept_string(
1050 + grammar,
1051 + getattr(cls, sample_name),
1052 + require_termination=False,
1053 + )
1054 + should_admit = sample_name in expected_admission_set
1055 + assert admitted is should_admit, (
1056 + f"Expected structured_outputs admission for {sample_name} "
1057 + f"to be {should_admit}, got {admitted}."
1058 + )
1059 +
1060 + @pytest.mark.parametrize("request_kind", ["chat", "responses"])
1061 + @pytest.mark.parametrize(
1062 + ("request_kwargs", "expected_admission"),
1063 + [
1064 + (
1065 + {"tool_choice": "auto", "strict_tools": True},
1066 + [
1067 + "COMMENTARY",
1068 + "TOOL_CALL_1",
1069 + "TOOL_CALL_2",
1070 + "FINAL_JSON_SCHEMA",
1071 + "FINAL_JSON_OBJECT",
1072 + "FINAL_TEXT_ONLY",
1073 + "FINAL_REGEX",
1074 + "FINAL_CHOICE",
1075 + "FINAL_GRAMMAR",
1076 + "FINAL_STRUCTURAL_TAG",
1077 + ],
1078 + ),
1079 + (
1080 + {"tool_choice": "required"},
1081 + ["COMMENTARY", "TOOL_CALL_1", "TOOL_CALL_2"],
1082 + ),
1083 + (
1084 + {"tool_choice": "named"},
1085 + ["COMMENTARY", "TOOL_CALL_2"],
1086 + ),
1087 + (
1088 + {"response_format_type": "json_schema"},
1089 + ["FINAL_JSON_SCHEMA"],
1090 + ),
1091 + (
1092 + {"response_format_type": "json_object"},
1093 + ["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
1094 + ),
1095 + (
1096 + {"structured_outputs": StructuredOutputsParams(json=OUTPUT_SCHEMA)},
1097 + ["FINAL_JSON_SCHEMA"],
1098 + ),
1099 + (
1100 + {"structured_outputs": StructuredOutputsParams(json_object=True)},
1101 + ["FINAL_JSON_SCHEMA", "FINAL_JSON_OBJECT"],
1102 + ),
1103 + (
1104 + {"structured_outputs": StructuredOutputsParams(regex=r"regex")},
1105 + ["FINAL_REGEX"],
1106 + ),
1107 + (
1108 + {
1109 + "structured_outputs": StructuredOutputsParams(
1110 + choice=["choice1", "choice2"]
1111 + )
1112 + },
1113 + ["FINAL_CHOICE"],
1114 + ),
1115 + (
1116 + {
1117 + "structured_outputs": StructuredOutputsParams(
1118 + grammar='root ::= "grammar"'
1119 + )
1120 + },
1121 + ["FINAL_GRAMMAR"],
1122 + ),
1123 + (
1124 + {
1125 + "structured_outputs": StructuredOutputsParams(
1126 + structural_tag=json.dumps(
1127 + {
1128 + "type": "structural_tag",
1129 + "format": {
1130 + "type": "json_schema",
1131 + "json_schema": OUTPUT_SCHEMA,
1132 + },
1133 + }
1134 + )
1135 + )
1136 + },
1137 + ["FINAL_JSON_SCHEMA"],
1138 + ),
1139 + (
1140 + {
1141 + "structured_outputs": StructuredOutputsParams(
1142 + structural_tag=json.dumps(
1143 + {
1144 + "type": "structural_tag",
1145 + "structures": [
1146 + {
1147 + "begin": "<tag>",
1148 + "schema": {"type": "object"},
1149 + "end": "</tag>",
1150 + }
1151 + ],
1152 + "triggers": ["<tag>"],
1153 + }
1154 + )
1155 + )
1156 + },
1157 + [
1158 + # Legacy triggered tags allow free text until a trigger, so
1159 + # unconstrained final-channel payloads are also admitted.
1160 + "FINAL_TEXT_ONLY",
1161 + "FINAL_REGEX",
1162 + "FINAL_CHOICE",
1163 + "FINAL_GRAMMAR",
1164 + "FINAL_STRUCTURAL_TAG",
1165 + ],
1166 + ),
1167 + ],
1168 + ids=[
1169 + "tool_auto_strict",
1170 + "tool_required",
1171 + "tool_named",
1172 + "response_format_json_schema",
1173 + "response_format_json_object",
1174 + "structured_outputs_json",
1175 + "structured_outputs_json_object",
1176 + "structured_outputs_regex",
1177 + "structured_outputs_choice",
1178 + "structured_outputs_grammar",
1179 + "structured_outputs_structural_tag_modern",
1180 + "structured_outputs_structural_tag_legacy",
1181 + ],
1182 + )
1183 + def test_adjust_request(
1184 + self,
1185 + harmony_parser,
1186 + request_kind,
1187 + request_kwargs,
1188 + expected_admission,
1189 + ):
1190 + request = self._build_request(request_kind, **request_kwargs)
1191 + adjusted_request = harmony_parser.adjust_request(request)
1192 + self._assert_format_cleared(adjusted_request)
1193 + self._assert_structured_outputs_admission(
1194 + adjusted_request,
1195 + expected_admission,
1196 + )
tests/reasoning/test_gptoss_reasoning_parser.py
+5 -344
@@ -1,351 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -import json
4 from unittest.mock import Mock
5
7 -import pytest
8 -from transformers import AutoTokenizer
6 +from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
7
10 -from vllm.entrypoints.mcp.tool_server import ToolServer
11 -from vllm.reasoning import ReasoningParser
12 -from vllm.reasoning.gptoss_reasoning_parser import (
13 - GptOssReasoningParser,
14 - from_builtin_tool_to_tag,
15 - no_func_reasoning_tag,
16 -)
8
18 -REASONING_MODEL_NAME = "openai/gpt-oss-120b"
19 -
20 -
21 -@pytest.fixture(scope="module")
22 -def gpt_oss_tokenizer():
23 - return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
24 -
25 -
26 -USER_MESSAGE_START = "<|start|>user<|message|>"
27 -REASONING_SECTION_START = "<|end|><|start|>assistant<|channel|>analysis<|message|>"
28 -END = "<|end|>"
29 -ASSISTANT_START = "<|start|>assistant"
30 -ASSISTANT_CONTENT_START_PREFIX = END + ASSISTANT_START + "<|channel|>final"
31 -ASSISTANT_CONTENT_START_SUFFIX = "<|message|>"
32 -ASSISTANT_CONTENT_START = (
33 - ASSISTANT_CONTENT_START_PREFIX + ASSISTANT_CONTENT_START_SUFFIX
34 -)
35 -
36 -BASIC_CONTENT = {
37 - "output": REASONING_SECTION_START
38 - + "This is reasoning"
39 - + ASSISTANT_CONTENT_START
40 - + "This is the rest",
41 - "is_reasoning_end": True,
42 -}
43 -
44 -BASIC_REASONING_ONLY = {
45 - "output": REASONING_SECTION_START + "This is reasoning" + "<|end|>",
46 - "is_reasoning_end": False,
47 -}
48 -BASIC_NO_REASONING_NO_ASSISTANT = {
49 - "output": USER_MESSAGE_START + "This is a user message",
50 - "is_reasoning_end": False,
51 -}
52 -
53 -# Edge-case where the model omits the assistant tag entirely.
54 -BASIC_NO_REASONING_ASSISTANT = {
55 - "output": USER_MESSAGE_START + "This is a user message<|end|><|channel|>final",
56 - "is_reasoning_end": True,
57 -}
58 -
59 -COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY = {
60 - "output": REASONING_SECTION_START
61 - + "This is reasoning"
62 - + ASSISTANT_CONTENT_START_PREFIX,
63 - "is_reasoning_end": False,
64 -}
65 -
66 -COMPLEX_CONTENT_SUFFIX_ONLY = {
67 - "output": REASONING_SECTION_START
68 - + "This is reasoning"
69 - + ASSISTANT_CONTENT_START_SUFFIX,
70 - "is_reasoning_end": False,
71 -}
72 -
73 -COMPLEX_CONTENT_1_NO_SUFFIX = {
74 - "output": REASONING_SECTION_START
75 - + "This is reasoning"
76 - + ASSISTANT_CONTENT_START_PREFIX
77 - + "<|constrain|> JSON ",
78 - "is_reasoning_end": False,
79 -}
80 -
81 -COMPLEX_CONTENT_1 = {
82 - "output": REASONING_SECTION_START
83 - + "This is reasoning"
84 - + ASSISTANT_CONTENT_START_PREFIX
85 - + "<|constrain|> JSON "
86 - + ASSISTANT_CONTENT_START_SUFFIX,
87 - "is_reasoning_end": True,
88 -}
89 -
90 -COMPLEX_CONTENT_1_WITH_CONTENT = {
91 - "output": REASONING_SECTION_START
92 - + "This is reasoning"
93 - + ASSISTANT_CONTENT_START_PREFIX
94 - + "<|constrain|> JSON "
95 - + ASSISTANT_CONTENT_START_SUFFIX
96 - + "This is the rest",
97 - "is_reasoning_end": True,
98 -}
99 -
100 -COMPLEX_CONTENT_2 = {
101 - "output": REASONING_SECTION_START
102 - + "This is reasoning"
103 - + ASSISTANT_CONTENT_START_PREFIX
104 - + "<|constrain|>ReplyAction "
105 - + ASSISTANT_CONTENT_START_SUFFIX
106 - + "This is the rest",
107 - "is_reasoning_end": True,
108 -}
109 -
110 -MULTI_TURN_CONTENT = {
111 - "output": USER_MESSAGE_START
112 - + "1st turn user message"
113 - + REASONING_SECTION_START
114 - + "1st turn reasoning"
115 - + ASSISTANT_CONTENT_START
116 - + "1st turn response"
117 - + END
118 - + USER_MESSAGE_START
119 - + "2nd turn user message"
120 - + END
121 - + ASSISTANT_START,
122 - "is_reasoning_end": False,
123 -}
124 -TEST_CASES = [
125 - BASIC_CONTENT,
126 - BASIC_REASONING_ONLY,
127 - COMPLEX_CONTENT_INCOMPLETE_PREFIX_ONLY,
128 - COMPLEX_CONTENT_SUFFIX_ONLY,
129 - COMPLEX_CONTENT_1_NO_SUFFIX,
130 - COMPLEX_CONTENT_1,
131 - COMPLEX_CONTENT_1_WITH_CONTENT,
132 - COMPLEX_CONTENT_2,
133 - MULTI_TURN_CONTENT,
134 -]
135 -
136 -
137 -@pytest.mark.parametrize(
138 - "output, is_reasoning_end",
139 - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
140 -)
141 -def test_gptoss_is_reasoning_end(
142 - output,
143 - is_reasoning_end,
144 - gpt_oss_tokenizer,
145 -):
146 - output = gpt_oss_tokenizer.tokenize(output)
147 - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
148 -
149 - # Test is_reasoning_end
150 - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(output)
151 - actual_is_reasoning_end = parser.is_reasoning_end(output_ids)
152 - assert is_reasoning_end == actual_is_reasoning_end
153 -
154 -
155 -class TestGptOssStructuralTags:
156 - """Test cases for GptOssReasoningParser structural tag functionality."""
157 -
158 - @pytest.fixture
159 - def mock_tokenizer(self):
160 - """Create a mock tokenizer for testing."""
161 - tokenizer = Mock()
162 - tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5])
163 - tokenizer.get_vocab = Mock(return_value={"<|end|>": 6})
164 - return tokenizer
165 -
166 - @pytest.fixture
167 - def reasoning_parser(self, mock_tokenizer):
168 - """Create a GptOssReasoningParser instance."""
169 - return GptOssReasoningParser(mock_tokenizer)
170 -
171 - def test_prepare_structured_tag_no_tool_server(self, reasoning_parser):
172 - """Test prepare_structured_tag with no tool server."""
173 - result = reasoning_parser.prepare_structured_tag(None, None)
174 - expected = json.dumps(no_func_reasoning_tag)
175 -
176 - assert result == expected
177 -
178 - # Verify the structure is correct
179 - parsed = json.loads(result)
180 - assert parsed["type"] == "structural_tag"
181 - assert parsed["format"]["type"] == "triggered_tags"
182 - assert len(parsed["format"]["tags"]) == 1
183 - assert parsed["format"]["tags"][0]["begin"] == "<|channel|>analysis<|message|>"
184 - assert parsed["format"]["triggers"] == ["<|channel|>analysis"]
185 -
186 - def test_prepare_structured_tag_with_original_tag(self, reasoning_parser):
187 - """Test prepare_structured_tag when original_tag is provided."""
188 - original_tag = '{"custom": "tag"}'
189 - result = reasoning_parser.prepare_structured_tag(original_tag, None)
190 -
191 - # Should return the original tag unchanged
192 - assert result == original_tag
193 -
194 - def test_from_builtin_tool_to_tag(self):
195 - """Test from_builtin_tool_to_tag function."""
196 - tags = from_builtin_tool_to_tag("python")
197 -
198 - assert len(tags) == 2
199 - assert tags[0]["begin"] == "<|channel|>commentary to=python"
200 - assert tags[0]["content"]["type"] == "any_text"
201 - assert tags[0]["end"] == "<|end|>"
202 -
203 - assert tags[1]["begin"] == "<|channel|>analysis to=python"
204 - assert tags[1]["content"]["type"] == "any_text"
205 - assert tags[1]["end"] == "<|end|>"
206 -
207 - @pytest.mark.parametrize(
208 - "tools",
209 - [
210 - [],
211 - ["browser"],
212 - ["python"],
213 - ["container"],
214 - ["browser", "python"],
215 - ["browser", "container"],
216 - ["python", "container"],
217 - ["browser", "python", "container"],
218 - ],
219 - )
220 - def test_json_validity_comprehensive(self, reasoning_parser, tools):
221 - """Test JSON validity across all possible tool combinations."""
222 - tool_server = Mock(spec=ToolServer)
223 - tool_server.has_tool = Mock(side_effect=lambda tool: tool in tools)
224 -
225 - result = reasoning_parser.prepare_structured_tag(None, tool_server)
226 - parsed_result = json.loads(result)
227 -
228 - assert parsed_result["type"] == "structural_tag"
229 - assert "format" in parsed_result
230 - assert "tags" in parsed_result["format"]
231 - assert "triggers" in parsed_result["format"]
232 -
233 - # Tag count should be: 1 (analysis) + 2 * len(tools)
234 - expected_tag_count = 1 + (2 * len(tools))
235 - assert len(parsed_result["format"]["tags"]) == expected_tag_count
236 -
237 - # Verify triggers are correctly configured
238 - expected_triggers = ["<|channel|>analysis"]
239 - if tools:
240 - expected_triggers.append("<|channel|>commentary to=")
241 - assert set(parsed_result["format"]["triggers"]) == set(expected_triggers)
242 -
243 - def test_no_cross_request_state_pollution(self, reasoning_parser):
244 - """Test that sequential calls with different tool servers produce
245 - independent results, guarding against shared mutable state
246 - (e.g. missing deepcopy in tag_with_builtin_funcs)."""
247 - tool_server_1 = Mock(spec=ToolServer)
248 - tool_server_1.has_tool = Mock(side_effect=lambda tool: tool == "python")
249 -
250 - tool_server_2 = Mock(spec=ToolServer)
251 - tool_server_2.has_tool = Mock(side_effect=lambda tool: tool == "browser")
252 -
253 - result_1 = reasoning_parser.prepare_structured_tag(None, tool_server_1)
254 - result_2 = reasoning_parser.prepare_structured_tag(None, tool_server_2)
255 -
256 - tags_1 = [tag["begin"] for tag in json.loads(result_1)["format"]["tags"]]
257 - tags_2 = [tag["begin"] for tag in json.loads(result_2)["format"]["tags"]]
258 -
259 - assert "<|channel|>commentary to=python" in tags_1
260 - assert "<|channel|>commentary to=browser" not in tags_1
261 -
262 - assert "<|channel|>commentary to=browser" in tags_2
263 - assert "<|channel|>commentary to=python" not in tags_2
264 -
265 - def test_tag_format_consistency(self, reasoning_parser):
266 - """Test that all generated tags follow consistent format,
267 - catching malformed tags from from_builtin_tool_to_tag."""
268 - tool_server = Mock(spec=ToolServer)
269 - tool_server.has_tool = Mock(
270 - side_effect=lambda tool: tool in ["python", "browser"]
271 - )
272 -
273 - result = reasoning_parser.prepare_structured_tag(None, tool_server)
274 - parsed_result = json.loads(result)
275 -
276 - for tag in parsed_result["format"]["tags"]:
277 - assert "begin" in tag
278 - assert "content" in tag
279 - assert "end" in tag
280 - assert tag["content"]["type"] == "any_text"
281 - assert tag["end"] == "<|end|>"
282 - assert tag["begin"].startswith("<|channel|>")
283 -
284 -
285 -@pytest.mark.parametrize(
286 - "output, is_reasoning_end",
287 - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
288 -)
289 -def test_gptoss_is_reasoning_end_streaming(
290 - output,
291 - is_reasoning_end,
292 - gpt_oss_tokenizer,
293 -):
294 - """Streaming override must agree with is_reasoning_end for all cases."""
295 - tokens = gpt_oss_tokenizer.tokenize(output)
296 - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
297 - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
298 - delta_ids = output_ids[-1:] if output_ids else []
299 - actual = parser.is_reasoning_end_streaming(output_ids, delta_ids)
300 - assert is_reasoning_end == actual
301 -
302 -
303 -@pytest.mark.parametrize(
304 - "output, is_reasoning_end",
305 - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
306 -)
307 -def test_gptoss_is_reasoning_end_streaming_long_prefix(
308 - output,
309 - is_reasoning_end,
310 - gpt_oss_tokenizer,
311 -):
312 - """Windowing must produce correct results even with a long prefix."""
313 - tokens = gpt_oss_tokenizer.tokenize(output)
314 - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
315 - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
316 - # Prepend 10k dummy reasoning tokens to simulate a long generation
317 - long_prefix = [1] * 10_000
318 - padded_ids = long_prefix + list(output_ids)
319 - delta_ids = output_ids[-1:] if output_ids else []
320 - actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
321 - assert is_reasoning_end == actual
322 -
323 -
324 -@pytest.mark.parametrize(
325 - "output, is_reasoning_end",
326 - [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES],
327 -)
328 -def test_gptoss_is_reasoning_end_streaming_large_delta(
329 - output,
330 - is_reasoning_end,
331 - gpt_oss_tokenizer,
332 -):
333 - """Simulate speculative decoding where the entire test sequence arrives
334 - as a single large delta appended after a long prefix. The window must
335 - expand to cover delta_ids so the end pattern is never missed."""
336 - tokens = gpt_oss_tokenizer.tokenize(output)
337 - parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer)
338 - output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens)
339 - long_prefix = [1] * 10_000
340 - padded_ids = long_prefix + list(output_ids)
341 - # delta_ids = the entire test sequence (as if accepted in one spec step)
342 - delta_ids = list(output_ids)
343 - actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids)
344 - assert is_reasoning_end == actual
345 -
346 -
347 -def test_gptoss_is_reasoning_end_streaming_signature(gpt_oss_tokenizer):
348 - """Verify the method is callable with the expected signature."""
349 - parser = GptOssReasoningParser(gpt_oss_tokenizer)
350 - result = parser.is_reasoning_end_streaming([], [])
351 - assert result is False
9 +def test_gptoss_reasoning_ended_is_true():
10 + parser = GptOssReasoningParser(Mock())
11 + assert parser.is_reasoning_end([]) is True
12 + assert parser.is_reasoning_end_streaming([], []) is True
tests/tool_parsers/test_structural_tag_registry.py
+2 -2
@@ -33,7 +33,7 @@ from vllm.tool_parsers.structural_tag_registry import (
33 SUPPORTED_STRUCTURAL_TAG_MODELS,
34 VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
35 XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS,
36 - _get_function_parameters,
36 + get_function_parameters,
37 get_model_structural_tag,
38 )
39
@@ -530,7 +530,7 @@ def test_get_function_parameters_relaxes_function_strict_false():
530 strict=False,
531 )
532
533 - assert _get_function_parameters(function) is True
533 + assert get_function_parameters(function) is True
534
535
536 def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]:
vllm/entrypoints/openai/responses/serving.py
+8 -5
@@ -749,11 +749,14 @@ class OpenAIServingResponses(GenerateBaseServing):
749 request: ResponsesRequest,
750 prev_response: ResponsesResponse | None,
751 ):
752 - if request.tool_choice not in ("auto", "none"):
753 - raise NotImplementedError(
754 - "Only 'auto' or 'none' tool_choice is supported "
755 - "in response API with Harmony"
756 - )
752 + if self.parser is not None:
753 + # HarmonyParser doesn't need chat_template_kwargs
754 + # TODO: Unify adjust_request() call with non-harmony branch
755 + self.parser(
756 + self.renderer.get_tokenizer(),
757 + request.tools,
758 + model_config=self.model_config,
759 + ).adjust_request(request=request)
760
761 arrival_time = time.time()
762 messages = self._construct_input_messages_with_harmony(request, prev_response)
vllm/parser/harmony.py
+230 -2
@@ -5,14 +5,31 @@ from __future__ import annotations
5
6 import json
7 from collections.abc import Sequence
8 -from dataclasses import dataclass
8 +from dataclasses import dataclass, replace
9 from enum import Enum, auto
10 from typing import TYPE_CHECKING, NamedTuple
11
12 from openai_harmony import HarmonyError, Message, Role
13 +from xgrammar import StructuralTag
14 +from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam
15 +from xgrammar.structural_tag import (
16 + AnyTextFormat,
17 + ConstStringFormat,
18 + Format,
19 + GrammarFormat,
20 + JSONSchemaFormat,
21 + OptionalFormat,
22 + OrFormat,
23 + RegexFormat,
24 + SequenceFormat,
25 + TagFormat,
26 + TriggeredTagsFormat,
27 +)
28
29 from vllm.entrypoints.chat_utils import make_tool_call_id
15 -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
30 +from vllm.entrypoints.openai.chat_completion.protocol import (
31 + ChatCompletionRequest,
32 +)
33 from vllm.entrypoints.openai.engine.protocol import (
34 DeltaFunctionCall,
35 DeltaMessage,
@@ -28,7 +45,13 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
45 from vllm.logger import init_logger
46 from vllm.parser.abstract_parser import DelegatingParser
47 from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
48 +from vllm.sampling_params import StructuredOutputsParams
49 from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
50 +from vllm.tool_parsers.structural_tag_registry import (
51 + SimplifiedToolChoice,
52 + get_function_parameters,
53 + register_vllm_structural_tag,
54 +)
55
56 if TYPE_CHECKING:
57 from openai_harmony import Message, StreamableParser
@@ -346,6 +369,12 @@ class HarmonyParser(DelegatingParser):
369 reasoning_token_count=reasoning_token_count,
370 )
371
372 + def adjust_request(
373 + self, request: ChatCompletionRequest | ResponsesRequest
374 + ) -> ChatCompletionRequest | ResponsesRequest:
375 + request = _adjust_output_format(request)
376 + return super().adjust_request(request)
377 +
378 @staticmethod
379 def _normalize_recipient(recipient: str | None) -> str | None:
380 """Remove constrained formats misparsed into recipients by older Harmony."""
@@ -356,3 +385,202 @@ class HarmonyParser(DelegatingParser):
385 if constrain_index == -1:
386 return recipient
387 return recipient[:constrain_index].rstrip() or None
388 +
389 +
390 +# Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|>
391 +# <|return|> is represented as "" since it's the default stop token, which xgrammar
392 +# disallows under constraints, leading to bad or infinite generation.
393 +# StreamableParser doesn't consider <|endoftext|> as a message end, so it's excluded
394 +# TODO: Remove <|call|> once #50595 lands.
395 +_END_TAG = ["<|end|>", "<|call|>", ""]
396 +_FINAL_BEGIN = "<|channel|>final{constrain}<|message|>"
397 +_TOOL_CALL_CHANNELS = [
398 + "<|channel|>commentary",
399 + "<|channel|>analysis",
400 + "<|channel|>final",
401 +]
402 +_FUNCTION_CALL_BEGINS = [
403 + "to=functions.{name} {channel} json<|message|>",
404 + "to=functions.{name} {channel} <|constrain|>json<|message|>",
405 + "{channel} to=functions.{name} json<|message|>",
406 + "{channel} to=functions.{name} <|constrain|>json<|message|>",
407 +]
408 +_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})
409 +_ANY_CONTENT = AnyTextFormat()
410 +
411 +
412 +def _assemble_tag(
413 + allow_analysis: bool, allow_commentary: bool, content: Format
414 +) -> StructuralTag:
415 + tags = []
416 + if allow_analysis:
417 + analysis_tag = OptionalFormat(
418 + content=SequenceFormat(
419 + elements=[
420 + TagFormat(
421 + begin="<|channel|>analysis<|message|>",
422 + content=_ANY_CONTENT,
423 + end="<|end|>",
424 + ),
425 + ConstStringFormat(value="<|start|>assistant"),
426 + ]
427 + )
428 + )
429 + tags.append(analysis_tag)
430 +
431 + if allow_commentary:
432 + commentary_tag = OptionalFormat(
433 + content=SequenceFormat(
434 + elements=[
435 + TagFormat(
436 + begin="<|channel|>commentary<|message|>",
437 + content=_ANY_CONTENT,
438 + end="<|end|>",
439 + ),
440 + ConstStringFormat(value="<|start|>assistant"),
441 + ]
442 + )
443 + )
444 + tags.append(commentary_tag)
445 +
446 + tags.append(content)
447 +
448 + return StructuralTag(format=SequenceFormat(elements=tags))
449 +
450 +
451 +@register_vllm_structural_tag("harmony")
452 +def get_harmony_structural_tag(
453 + tools: list[FunctionToolParam],
454 + builtin_tools: list[BuiltinToolParam],
455 + tool_choice: SimplifiedToolChoice,
456 + reasoning: bool,
457 +) -> StructuralTag:
458 + # reasoning always enabled for Harmony
459 + del reasoning
460 +
461 + if builtin_tools:
462 + # Fallback for built-in tools
463 + tags = [
464 + TagFormat(
465 + begin="to=",
466 + content=AnyTextFormat(excludes=["<|start|>"]),
467 + end=_END_TAG,
468 + )
469 + ]
470 + tags.extend(
471 + TagFormat(
472 + begin=channel + " to=",
473 + content=AnyTextFormat(excludes=["<|start|>", "<|channel|>"]),
474 + end=_END_TAG,
475 + )
476 + for channel in _TOOL_CALL_CHANNELS
477 + )
478 + else:
479 + tags = [
480 + TagFormat(
481 + begin=pattern.format(name=tool.function.name, channel=channel),
482 + content=JSONSchemaFormat(
483 + json_schema=get_function_parameters(tool.function)
484 + ),
485 + end=_END_TAG,
486 + )
487 + for tool in tools
488 + for pattern in _FUNCTION_CALL_BEGINS
489 + for channel in _TOOL_CALL_CHANNELS
490 + ]
491 +
492 + if tool_choice == "auto":
493 + tags.append(
494 + TagFormat(
495 + begin=_FINAL_BEGIN.format(constrain=" <|constrain|>json"),
496 + content=_ANY_CONTENT,
497 + end=_END_TAG,
498 + )
499 + )
500 + tags.append(
501 + TagFormat(
502 + begin=_FINAL_BEGIN.format(constrain=""),
503 + content=_ANY_CONTENT,
504 + end=_END_TAG,
505 + )
506 + )
507 +
508 + return _assemble_tag(
509 + allow_analysis=True, allow_commentary=True, content=OrFormat(elements=tags)
510 + )
511 +
512 +
513 +def _params_to_final_content(params: StructuredOutputsParams) -> Format | None:
514 + """Map StructuredOutputsParams in a XGrammar Format."""
515 + if params.json_object:
516 + return _JSON_CONTENT
517 + if params.json is not None:
518 + schema = params.json
519 + if isinstance(schema, str):
520 + schema = json.loads(schema)
521 + return JSONSchemaFormat(json_schema=schema)
522 + if params.regex is not None:
523 + return RegexFormat(pattern=params.regex)
524 + if params.choice is not None:
525 + return OrFormat(
526 + elements=[ConstStringFormat(value=choice) for choice in params.choice]
527 + )
528 + if params.grammar is not None:
529 + return GrammarFormat(grammar=params.grammar)
530 + if params.structural_tag is not None:
531 + s_tag = json.loads(params.structural_tag)
532 + if "structures" in s_tag:
533 + # LegacyStructuralTagResponseFormat
534 + return TriggeredTagsFormat(
535 + triggers=s_tag["triggers"],
536 + tags=[
537 + TagFormat(
538 + begin=structure["begin"],
539 + content=JSONSchemaFormat(json_schema=structure["schema"]),
540 + end=structure["end"],
541 + )
542 + for structure in s_tag["structures"]
543 + ],
544 + )
545 + # StructuralTagResponseFormat
546 + return StructuralTag.model_validate(s_tag).format
547 + return None
548 +
549 +
550 +def _adjust_output_format(
551 + request: ChatCompletionRequest | ResponsesRequest,
552 +) -> ChatCompletionRequest | ResponsesRequest:
553 + """Canonicalize request constraints into a reasoning-aware StructuralTag."""
554 + params = request.extract_structured_outputs()
555 + if params is None:
556 + return request
557 +
558 + final_content = _params_to_final_content(params)
559 + if final_content is None:
560 + return request
561 +
562 + if isinstance(final_content, JSONSchemaFormat):
563 + begin = _FINAL_BEGIN.format(constrain=" <|constrain|>json")
564 + else:
565 + begin = _FINAL_BEGIN.format(constrain="")
566 +
567 + structural_tag = _assemble_tag(
568 + allow_analysis=True,
569 + allow_commentary=False,
570 + content=TagFormat(begin=begin, content=final_content, end=_END_TAG),
571 + )
572 +
573 + request.structured_outputs = replace(
574 + params,
575 + json=None,
576 + regex=None,
577 + choice=None,
578 + grammar=None,
579 + json_object=None,
580 + structural_tag=json.dumps(structural_tag.model_dump()),
581 + )
582 + if isinstance(request, ResponsesRequest):
583 + request.text = None
584 + else:
585 + request.response_format = None
586 + return request
vllm/reasoning/gptoss_reasoning_parser.py
+2 -130
@@ -1,65 +1,17 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import json
3 from collections.abc import Iterable, Sequence
4 from typing import TYPE_CHECKING
5
6 from transformers import PreTrainedTokenizerBase
7
9 -from vllm.entrypoints.mcp.tool_server import ToolServer
8 from vllm.entrypoints.openai.engine.protocol import DeltaMessage
11 -from vllm.logger import init_logger
9 from vllm.reasoning import ReasoningParser
10
11 if TYPE_CHECKING:
12 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
13 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
14
18 -logger = init_logger(__name__)
19 -
20 -no_func_reasoning_tag = {
21 - "type": "structural_tag",
22 - "format": {
23 - "type": "triggered_tags",
24 - "tags": [
25 - {
26 - "begin": "<|channel|>analysis<|message|>",
27 - "content": {"type": "any_text"},
28 - "end": "<|end|>",
29 - }
30 - ],
31 - "triggers": ["<|channel|>analysis"],
32 - "stop_after_first": False,
33 - },
34 -}
35 -
36 -
37 -def from_builtin_tool_to_tag(tool: str) -> list[dict]:
38 - tag = [
39 - {
40 - "begin": f"<|channel|>commentary to={tool}",
41 - "content": {"type": "any_text"},
42 - "end": "<|end|>",
43 - },
44 - {
45 - "begin": f"<|channel|>analysis to={tool}",
46 - "content": {"type": "any_text"},
47 - "end": "<|end|>",
48 - },
49 - ]
50 - return tag
51 -
52 -
53 -def tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list: list[str]) -> dict:
54 - import copy
55 -
56 - new_tag = copy.deepcopy(no_func_reasoning_tag)
57 - new_tag["format"]["triggers"].append("<|channel|>commentary to=")
58 -
59 - for tool in builtin_tool_list:
60 - new_tag["format"]["tags"].extend(from_builtin_tool_to_tag(tool))
61 - return new_tag
62 -
15
16 class GptOssReasoningParser(ReasoningParser):
17 """
@@ -71,64 +23,14 @@ class GptOssReasoningParser(ReasoningParser):
23
24 def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs):
25 super().__init__(tokenizer, *args, **kwargs)
74 - # The model can output some special tokens between "final" and "<|message|>"
75 - # So we need to look for both sequences to determine the end of reasoning.
76 - self.reasoning_end_token_ids_prefix = self.model_tokenizer.encode(
77 - "<|channel|>final"
78 - )
79 - self.reasoning_end_token_ids_suffix = self.model_tokenizer.encode("<|message|>")
80 - # We also need to check for the <|end|> token to avoid false positives from
81 - # previous messages in multi-turn conversations.
82 - self.eom_token_id = self.vocab["<|end|>"]
83 - self.reasoning_max_num_between_tokens = 20
26
27 def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
86 - end_token_ids_prefix = self.reasoning_end_token_ids_prefix
87 - end_token_ids_suffix = self.reasoning_end_token_ids_suffix
88 - assert len(end_token_ids_prefix) > 0, "reasoning_end_token_ids_prefix is empty"
89 - assert len(end_token_ids_suffix) > 0, "reasoning_end_token_ids_suffix is empty"
90 - # Check if the end sequence is present in the input_ids.
91 - # We search from the end of input_ids to find the last match.
92 - for i in range(len(input_ids) - len(end_token_ids_prefix), -1, -1):
93 - if input_ids[i] == self.eom_token_id:
94 - # We looped backwards far enough to find the end of a previous message,
95 - # which means we have searched the entirety of the current message
96 - # and can exit early without searching further back into prior
97 - # messages of the conversation.
98 - return False
99 - if input_ids[i : i + len(end_token_ids_prefix)] == end_token_ids_prefix:
100 - # We have found the prefix, now we look for the suffix after the prefix.
101 - suffix_start = i + len(end_token_ids_prefix)
102 - for j in range(
103 - suffix_start, len(input_ids) - len(end_token_ids_suffix) + 1
104 - ):
105 - if j - suffix_start >= self.reasoning_max_num_between_tokens:
106 - break
107 - if (
108 - input_ids[j : j + len(end_token_ids_suffix)]
109 - == end_token_ids_suffix
110 - ):
111 - return True
112 - return False
28 + return True
29
30 def is_reasoning_end_streaming(
31 self, input_ids: Sequence[int], delta_ids: Iterable[int]
32 ) -> bool:
117 - # The pattern window covers the end-of-reasoning marker itself.
118 - # We add len(delta_ids) so that under speculative decoding (where
119 - # a single step can accept many tokens) the entire accepted chunk
120 - # is always inside the scan region.
121 - delta_ids = tuple(delta_ids)
122 - pattern_len = (
123 - len(self.reasoning_end_token_ids_prefix)
124 - + self.reasoning_max_num_between_tokens
125 - + len(self.reasoning_end_token_ids_suffix)
126 - )
127 - window = pattern_len + len(delta_ids)
128 - n = len(input_ids)
129 - if n <= window:
130 - return self.is_reasoning_end(input_ids)
131 - return self.is_reasoning_end(input_ids[n - window :])
33 + return True
34
35 def extract_content_ids(self, input_ids: list[int]) -> list[int]:
36 raise NotImplementedError(
@@ -159,33 +61,3 @@ class GptOssReasoningParser(ReasoningParser):
61 "GptOssReasoningParser only provides boundary detection. "
62 "Use HarmonyParser for output parsing."
63 )
162 -
163 - # This function prepares the structural tag to format reasoning output
164 - def prepare_structured_tag(
165 - self, original_tag: str | None, tool_server: ToolServer | None
166 - ) -> str | None:
167 - if original_tag is None:
168 - if tool_server is None:
169 - return json.dumps(no_func_reasoning_tag)
170 - else:
171 - builtin_tool_list: list[str] = []
172 - if tool_server.has_tool("browser"):
173 - builtin_tool_list.append("browser")
174 - if tool_server.has_tool("python"):
175 - builtin_tool_list.append("python")
176 - if tool_server.has_tool("container"):
177 - builtin_tool_list.append("container")
178 -
179 - if len(builtin_tool_list) > 0:
180 - logger.info("Builtin_tool_list: %s", builtin_tool_list)
181 - func_tag = json.dumps(
182 - tag_with_builtin_funcs(no_func_reasoning_tag, builtin_tool_list)
183 - )
184 - else:
185 - logger.info("Builtin_tool_list is empty")
186 - func_tag = json.dumps(no_func_reasoning_tag)
187 -
188 - return func_tag
189 - else:
190 - # There is potential risk for appending the tag to the original tag
191 - return original_tag
vllm/renderers/online_renderer.py
+9
@@ -201,6 +201,15 @@ class OnlineRenderer:
201 )
202 else:
203 # For GPT-OSS.
204 + if self.parser is not None:
205 + # HarmonyParser doesn't need chat_template_kwargs
206 + # TODO: Unify adjust_request() call with non-harmony branch
207 + self.parser(
208 + self.renderer.get_tokenizer(),
209 + request.tools,
210 + model_config=self.model_config,
211 + ).adjust_request(request=request)
212 +
213 should_include_tools = tool_dicts is not None
214 conversation, engine_inputs = self._make_request_with_harmony(
215 request, should_include_tools
vllm/tool_parsers/gptoss_tool_parser.py
+2
@@ -22,6 +22,8 @@ class GptOssToolParser(ToolParser):
22 capability declaration via HarmonyParser.tool_parser_cls.
23 """
24
25 + structural_tag_model = "harmony"
26 +
27 def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
28 super().__init__(tokenizer, tools)
29
vllm/tool_parsers/structural_tag_registry.py
+4 -5
@@ -65,7 +65,6 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
65 "qwen_3_5",
66 "qwen_3_coder",
67 "qwen_3",
68 - "harmony",
68 "deepseek_v3_2",
69 "glm_4_7",
70 "deepseek_v4",
@@ -209,7 +208,7 @@ def _dump_allowed_tool_ref_for_xgrammar(tool_ref: AllowedToolRef) -> AllowedTool
208 return tool_ref
209
210
212 -def _get_function_parameters(function) -> dict[str, Any] | bool:
211 +def get_function_parameters(function) -> dict[str, Any] | bool:
212 if getattr(function, "strict", None) is False:
213 return True
214 return function.parameters if function.parameters is not None else True
@@ -230,7 +229,7 @@ def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
229 TagFormat(
230 begin=begin + tool.function.name + arguments_field_prefix,
231 content=JSONSchemaFormat(
233 - json_schema=_get_function_parameters(tool.function)
232 + json_schema=get_function_parameters(tool.function)
233 ),
234 end=end,
235 )
@@ -279,7 +278,7 @@ def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
278 TagFormat(
279 begin=f'<invoke name="{tool.function.name}">\n',
280 content=JSONSchemaFormat(
282 - json_schema=_get_function_parameters(tool.function),
281 + json_schema=get_function_parameters(tool.function),
282 style="minimax_xml",
283 ),
284 end="</invoke>\n",
@@ -537,7 +536,7 @@ def _k3_arguments_block(parameters: dict[str, Any] | bool) -> Any:
536 def _k3_call_tag(tool: FunctionToolParam) -> TagFormat:
537 """One ``call`` tag: ``<|open|>call tool="N" index="<digits>"<|sep|> args``."""
538 function = tool.function
540 - parameters = _get_function_parameters(function)
539 + parameters = get_function_parameters(function)
540 begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="'
541 return TagFormat(
542 begin=begin,