@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f712fd0d7d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)

Signed-off-by: Yifan Zong <yzong@redhat.com>

yzong-rh committed Jun 11, 2026 at 17:18 UTC f712fd0d7db6e0b2c7fbdb6e77cae155c81fd8c5
16 files changed +822 -816
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+38 -10
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
38 from vllm.exceptions import VLLMValidationError
39 from vllm.inputs import TokensPrompt
40 from vllm.outputs import CompletionOutput, RequestOutput
41 +from vllm.parser import HarmonyParser
42 from vllm.renderers.hf import HfRenderer
43 from vllm.renderers.mistral import MistralRenderer
44 from vllm.tokenizers import get_tokenizer
45 from vllm.tokenizers.mistral import MistralTokenizer
46 from vllm.tokenizers.registry import cached_tokenizer_from_config
46 -from vllm.tool_parsers import ToolParserManager
47 from vllm.v1.engine.async_llm import AsyncLLM
48
49 GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
@@ -575,7 +575,13 @@ def _build_serving_render(
575 )
576
577
578 -def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
578 +def _build_serving_chat(
579 + engine: AsyncLLM,
580 + *,
581 + reasoning_parser: str = "",
582 + tool_parser: str | None = None,
583 + enable_auto_tools: bool = False,
584 +) -> OpenAIServingChat:
585 models = OpenAIServingModels(
586 engine_client=engine,
587 base_model_paths=BASE_MODEL_PATHS,
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
596 chat_template=CHAT_TEMPLATE,
597 chat_template_content_format="auto",
598 request_logger=None,
599 + reasoning_parser=reasoning_parser,
600 + tool_parser=tool_parser,
601 + enable_auto_tools=enable_auto_tools,
602 )
603
604 return serving_chat
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
646 serving_chat = _build_serving_chat(mock_engine)
647 messages = [{"role": "user", "content": "what is 1+1?"}]
648
640 - async def return_model_name(*args):
649 + async def return_model_name(*args, **kwargs):
650 return args[3]
651
652 serving_chat.chat_completion_full_generator = return_model_name
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
1219 mock_engine = MagicMock(spec=AsyncLLM)
1220 mock_engine.errored = False
1221 mock_engine.model_config = MockModelConfig()
1222 + mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
1223 + mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
1224 mock_engine.input_processor = MagicMock()
1225 mock_engine.renderer = _build_renderer(mock_engine.model_config)
1226 return mock_engine
1227
1228 @pytest.fixture()
1229 def serving_chat(self, mock_engine) -> OpenAIServingChat:
1219 - chat = _build_serving_chat(mock_engine)
1220 - chat.use_harmony = True
1221 - chat.tool_parser = ToolParserManager.get_tool_parser("openai")
1230 + chat = _build_serving_chat(
1231 + mock_engine,
1232 + reasoning_parser="openai_gptoss",
1233 + tool_parser="openai",
1234 + enable_auto_tools=True,
1235 + )
1236 + assert chat.parser_cls is HarmonyParser
1237 return chat
1238
1239 def mock_request_output_from_req_and_token_ids(
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
1292 stream: bool = False,
1293 ) -> ChatCompletionResponse:
1294 harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
1295 + tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
1296
1297 async def result_generator():
1298 if stream:
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
1320 request_id=req.request_id,
1321 model_name=req.model,
1322 conversation=[],
1307 - tokenizer=get_tokenizer(req.model),
1323 + tokenizer=tokenizer,
1324 request_metadata=RequestResponseMetadata(
1325 request_id=req.request_id,
1326 model_name=req.model,
1327 ),
1328 + chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
1329 )
1330
1331 if stream:
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
1333 return await result
1334
1335 @pytest.mark.asyncio
1319 - async def test_simple_chat(self, serving_chat, stream):
1336 + @pytest.mark.parametrize(
1337 + "include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
1338 + )
1339 + async def test_simple_chat(self, serving_chat, stream, include_reasoning):
1340 messages = [{"role": "user", "content": "what is 1+1?"}]
1341
1342 # Test the Harmony messages for the first turn's input
1323 - req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1343 + req = ChatCompletionRequest(
1344 + model=MODEL_NAME,
1345 + messages=messages,
1346 + include_reasoning=include_reasoning,
1347 + )
1348 input_messages, _ = (
1349 serving_chat.openai_serving_render._make_request_with_harmony(req)
1350 )
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
1366 response = await self.generate_response_from_harmony_str(
1367 serving_chat, req, response_str, stream=stream
1368 )
1345 - verify_chat_response(response, content=final_str, reasoning=reasoning_str)
1369 + verify_chat_response(
1370 + response,
1371 + content=final_str,
1372 + reasoning=reasoning_str if include_reasoning else None,
1373 + )
1374
1375 # Add the output messages from the first turn as input to the second turn
1376 for choice in response.choices:
tests/entrypoints/openai/parser/test_harmony_utils.py
-106
@@ -11,12 +11,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
11 auto_drop_analysis_messages,
12 create_tool_definition,
13 extract_function_from_recipient,
14 - get_encoding,
14 get_system_message,
15 has_custom_tools,
16 is_function_recipient,
17 parse_chat_input_to_harmony_message,
19 - parse_chat_output,
18 )
19 from vllm.entrypoints.openai.responses.harmony import (
20 response_input_to_harmony,
@@ -941,110 +939,6 @@ class TestAutoDropAnalysisMessages:
939 assert cleaned_messages == messages[1:]
940
941
944 -class TestParseChatOutput:
945 - def test_parse_chat_output_interrupted_first_message(self) -> None:
946 - harmony_str = "<|channel|>final<|message|>I'm in the middle of answering"
947 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
948 - reasoning, final_content, _ = parse_chat_output(token_ids)
949 - assert reasoning is None
950 - assert final_content == "I'm in the middle of answering"
951 -
952 - def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None:
953 - harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking"
954 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
955 - reasoning, final_content, _ = parse_chat_output(token_ids)
956 - assert reasoning == "I'm in the middle of thinking"
957 - assert final_content is None
958 -
959 - def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None:
960 - harmony_str = (
961 - "<|channel|>analysis<|message|>I'm thinking.<|end|>"
962 - "<|start|>assistant<|channel|>final"
963 - "<|message|>I'm in the middle of answering"
964 - )
965 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
966 - reasoning, final_content, _ = parse_chat_output(token_ids)
967 - assert reasoning == "I'm thinking."
968 - assert final_content == "I'm in the middle of answering"
969 -
970 - def test_parse_chat_output_complete_content(self) -> None:
971 - harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>"
972 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
973 - reasoning, final_content, _ = parse_chat_output(token_ids)
974 - assert reasoning is None
975 - assert final_content == "The answer is 4."
976 -
977 - def test_parse_chat_output_complete_commentary(self) -> None:
978 - harmony_str = (
979 - "<|channel|>commentary<|message|>I need to call some tools.<|end|>"
980 - )
981 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
982 - reasoning, final_content, _ = parse_chat_output(token_ids)
983 - assert reasoning is None
984 - assert final_content == "I need to call some tools."
985 -
986 - def test_parse_chat_output_complete_reasoning(self) -> None:
987 - harmony_str = (
988 - "<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
989 - )
990 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
991 - reasoning, final_content, _ = parse_chat_output(token_ids)
992 - assert reasoning == "I've thought hard about this."
993 - assert final_content is None
994 -
995 - def test_parse_chat_output_complete_reasoning_and_content(self) -> None:
996 - harmony_str = (
997 - "<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
998 - "<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>"
999 - )
1000 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
1001 - reasoning, final_content, _ = parse_chat_output(token_ids)
1002 - assert reasoning == "I've thought hard about this."
1003 - assert final_content == "The answer is 4."
1004 -
1005 - def test_parse_chat_output_commentary_with_recipient_excluded(self) -> None:
1006 - """Commentary with a recipient (tool call) should not appear in
1007 - final_content — those are handled separately by the tool parser.
1008 -
1009 - The first message is a preamble (visible), the second is a tool
1010 - call (excluded). Only the preamble should appear in final_content.
1011 - """
1012 - harmony_str = (
1013 - "<|channel|>commentary"
1014 - "<|message|>Let me check the weather.<|end|>"
1015 - "<|start|>assistant to=functions.get_weather"
1016 - "<|channel|>commentary"
1017 - '<|message|>{"location": "SF"}<|end|>'
1018 - )
1019 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
1020 - reasoning, final_content, _ = parse_chat_output(token_ids)
1021 - assert reasoning is None
1022 - assert final_content == "Let me check the weather."
1023 -
1024 - def test_parse_chat_output_interrupted_preamble(self) -> None:
1025 - """Partial/interrupted preamble (commentary without recipient) should
1026 - appear in final_content, not reasoning."""
1027 - harmony_str = "<|channel|>commentary<|message|>I'll search for that"
1028 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
1029 - reasoning, final_content, _ = parse_chat_output(token_ids)
1030 - assert reasoning is None
1031 - assert final_content == "I'll search for that"
1032 -
1033 - def test_parse_chat_output_preamble_then_final(self) -> None:
1034 - """Preamble followed by a final message should both appear in
1035 - final_content, joined by newline."""
1036 - harmony_str = (
1037 - "<|channel|>commentary"
1038 - "<|message|>Let me look that up.<|end|>"
1039 - "<|start|>assistant<|channel|>final"
1040 - "<|message|>The answer is 42.<|end|>"
1041 - )
1042 - token_ids = get_encoding().encode(harmony_str, allowed_special="all")
1043 - reasoning, final_content, _ = parse_chat_output(token_ids)
1044 - assert reasoning is None
1045 - assert final_content == "Let me look that up.\nThe answer is 42."
1046 -
1047 -
942 def test_has_custom_tools() -> None:
943 assert not has_custom_tools(set())
944 assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})
tests/parser/test_harmony.py new
+452
@@ -0,0 +1,452 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +from collections.abc import Sequence
6 +
7 +import pytest
8 +from openai_harmony import (
9 + Conversation,
10 + Message,
11 + RenderConversationConfig,
12 + Role,
13 +)
14 +from transformers import AutoTokenizer
15 +
16 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
17 +from vllm.entrypoints.openai.engine.protocol import FunctionCall
18 +from vllm.entrypoints.openai.parser.harmony_utils import (
19 + get_encoding,
20 +)
21 +from vllm.parser.harmony import HarmonyParser
22 +from vllm.parser.parser_manager import ParserManager
23 +
24 +REASONING_MODEL_NAME = "openai/gpt-oss-20b"
25 +
26 +
27 +@pytest.fixture(scope="module")
28 +def gpt_oss_tokenizer():
29 + return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
30 +
31 +
32 +@pytest.fixture
33 +def harmony_parser(gpt_oss_tokenizer):
34 + parser_cls = ParserManager.get_parser(
35 + tool_parser_name="openai",
36 + reasoning_parser_name="openai_gptoss",
37 + enable_auto_tools=True,
38 + model_name=REASONING_MODEL_NAME,
39 + is_harmony=True,
40 + )
41 + assert parser_cls is HarmonyParser
42 + return parser_cls(gpt_oss_tokenizer)
43 +
44 +
45 +@pytest.fixture
46 +def chat_request():
47 + return ChatCompletionRequest(
48 + model="openai/gpt-oss-20b",
49 + messages=[{"role": "user", "content": "Hello"}],
50 + )
51 +
52 +
53 +def encode_output(harmony_str: str) -> list[int]:
54 + return get_encoding().encode(harmony_str, allowed_special="all")
55 +
56 +
57 +def assistant(content: str, channel: str) -> Message:
58 + return Message.from_role_and_content(Role.ASSISTANT, content).with_channel(channel)
59 +
60 +
61 +def tool_call(
62 + recipient: str,
63 + content: str,
64 + channel: str = "commentary",
65 + content_type: str | None = "json",
66 +) -> Message:
67 + message = assistant(content, channel).with_recipient(recipient)
68 + return message if content_type is None else message.with_content_type(content_type)
69 +
70 +
71 +def get_model_output_tokens(
72 + prompt_messages: Sequence[Message],
73 + response_messages: Sequence[Message],
74 +) -> list[int]:
75 + enc = get_encoding()
76 + # Keep analysis messages when synthesizing model-output-only token sequences
77 + # for parser tests; the default render path drops them after a later final turn.
78 + config = RenderConversationConfig(auto_drop_analysis=False)
79 + prompt_ids = enc.render_conversation_for_completion(
80 + Conversation.from_messages(list(prompt_messages)),
81 + Role.ASSISTANT,
82 + config=config,
83 + )
84 + full_ids = enc.render_conversation_for_completion(
85 + Conversation.from_messages([*prompt_messages, *response_messages]),
86 + Role.ASSISTANT,
87 + config=config,
88 + )
89 + assert full_ids[: len(prompt_ids)] == prompt_ids
90 + return full_ids[len(prompt_ids) :]
91 +
92 +
93 +def get_text(msg: Message) -> str:
94 + return msg.content[0].text if msg.content else ""
95 +
96 +
97 +def visible_segments(result) -> list[tuple[str | None, str | None, str]]:
98 + return [
99 + (segment.channel, segment.recipient, segment.delta)
100 + for segment in result.segments
101 + if not segment.is_boundary and segment.delta
102 + ]
103 +
104 +
105 +def tool_call_tuples(tool_calls: list[FunctionCall] | None) -> list[tuple[str, str]]:
106 + return [] if tool_calls is None else [(tc.name, tc.arguments) for tc in tool_calls]
107 +
108 +
109 +class TestParse:
110 + # Rendered conversation outputs.
111 +
112 + def test_reasoning_only(self, harmony_parser, chat_request):
113 + prompt = [Message.from_role_and_content(Role.USER, "Why?")]
114 + response = [assistant("This is reasoning", "analysis")]
115 +
116 + reasoning, content, tool_calls = harmony_parser.parse(
117 + "",
118 + chat_request,
119 + model_output_token_ids=get_model_output_tokens(prompt, response),
120 + )
121 +
122 + assert reasoning == "This is reasoning"
123 + assert content is None
124 + assert tool_calls is None
125 +
126 + def test_content_only(self, harmony_parser, chat_request):
127 + prompt = [Message.from_role_and_content(Role.USER, "Hello")]
128 + response = [assistant("This is a test", "final")]
129 +
130 + reasoning, content, tool_calls = harmony_parser.parse(
131 + "",
132 + chat_request,
133 + model_output_token_ids=get_model_output_tokens(prompt, response),
134 + )
135 +
136 + assert reasoning is None
137 + assert content == "This is a test"
138 + assert tool_calls is None
139 +
140 + def test_reasoning_and_content(self, harmony_parser, chat_request):
141 + prompt = [Message.from_role_and_content(Role.USER, "What is 2+2?")]
142 + response = [
143 + assistant("I should think first.", "analysis"),
144 + assistant("The answer is 4.", "final"),
145 + ]
146 +
147 + reasoning, content, tool_calls = harmony_parser.parse(
148 + "",
149 + chat_request,
150 + model_output_token_ids=get_model_output_tokens(prompt, response),
151 + )
152 +
153 + assert reasoning == "I should think first."
154 + assert content == "The answer is 4."
155 + assert tool_calls is None
156 +
157 + @pytest.mark.parametrize(
158 + "tool_args",
159 + [
160 + '{"location": "Tokyo"}',
161 + '{\n"location": "Tokyo"\n}',
162 + ],
163 + )
164 + @pytest.mark.parametrize("tool_channel", ["commentary", "analysis"])
165 + def test_single_tool_call(
166 + self, harmony_parser, chat_request, tool_args, tool_channel
167 + ):
168 + prompt = [
169 + Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?")
170 + ]
171 + response = [tool_call("functions.get_current_weather", tool_args, tool_channel)]
172 +
173 + reasoning, content, tool_calls = harmony_parser.parse(
174 + "",
175 + chat_request,
176 + model_output_token_ids=get_model_output_tokens(prompt, response),
177 + )
178 +
179 + assert reasoning is None
180 + assert content is None
181 + assert tool_call_tuples(tool_calls) == [
182 + ("get_current_weather", json.dumps({"location": "Tokyo"}))
183 + ]
184 +
185 + def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request):
186 + prompt = [
187 + Message.from_role_and_content(
188 + Role.USER, "What is the weather in Tokyo based on where I'm at?"
189 + )
190 + ]
191 + response = [
192 + tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
193 + tool_call("functions.get_user_location", '{"location": "Tokyo"}'),
194 + tool_call(
195 + "functions.no_content_type",
196 + '{"location": "Tokyo"}',
197 + content_type=None,
198 + ),
199 + tool_call("functions.not_json_no_content_type", "foo", content_type=None),
200 + tool_call("functions.empty_args", "{}"),
201 + tool_call("functions.no_args", ""),
202 + ]
203 +
204 + _, content, tool_calls = harmony_parser.parse(
205 + "",
206 + chat_request,
207 + model_output_token_ids=get_model_output_tokens(prompt, response),
208 + )
209 +
210 + assert content is None
211 + assert tool_call_tuples(tool_calls) == [
212 + ("get_current_weather", json.dumps({"location": "Tokyo"})),
213 + ("get_user_location", json.dumps({"location": "Tokyo"})),
214 + ("no_content_type", json.dumps({"location": "Tokyo"})),
215 + ("not_json_no_content_type", "foo"),
216 + ("empty_args", json.dumps({})),
217 + ("no_args", ""),
218 + ]
219 +
220 + def test_tool_call_bare_recipient(self, harmony_parser, chat_request):
221 + prompt = [Message.from_role_and_content(Role.USER, "Weather?")]
222 + response = [tool_call("get_current_weather", '{"location": "Tokyo"}')]
223 +
224 + _, _, tool_calls = harmony_parser.parse(
225 + "",
226 + chat_request,
227 + model_output_token_ids=get_model_output_tokens(prompt, response),
228 + )
229 +
230 + assert tool_call_tuples(tool_calls) == [
231 + ("get_current_weather", json.dumps({"location": "Tokyo"}))
232 + ]
233 +
234 + def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request):
235 + prompt = [Message.from_role_and_content(Role.USER, "Use both tools.")]
236 + response = [
237 + tool_call("get_current_weather", '{"location": "Tokyo"}'),
238 + tool_call("get_user_location", "{}"),
239 + ]
240 +
241 + _, _, tool_calls = harmony_parser.parse(
242 + "",
243 + chat_request,
244 + model_output_token_ids=get_model_output_tokens(prompt, response),
245 + )
246 +
247 + assert tool_call_tuples(tool_calls) == [
248 + ("get_current_weather", json.dumps({"location": "Tokyo"})),
249 + ("get_user_location", json.dumps({})),
250 + ]
251 +
252 + def test_assistant_recipient_not_tool(self, harmony_parser, chat_request):
253 + prompt = [Message.from_role_and_content(Role.USER, "Hello")]
254 + response = [
255 + tool_call("assistant", "Some tool response", content_type=None),
256 + assistant("Here is the answer", "final"),
257 + ]
258 +
259 + reasoning, content, tool_calls = harmony_parser.parse(
260 + "",
261 + chat_request,
262 + model_output_token_ids=get_model_output_tokens(prompt, response),
263 + )
264 +
265 + assert reasoning is None
266 + assert content == "Here is the answer"
267 + assert tool_calls is None
268 +
269 + def test_tool_call_dotted_name(self, harmony_parser, chat_request):
270 + prompt = [Message.from_role_and_content(Role.USER, "Compute 2+3")]
271 + response = [tool_call("math.sum", '{"a": 2, "b": 3}')]
272 +
273 + _, _, tool_calls = harmony_parser.parse(
274 + "",
275 + chat_request,
276 + model_output_token_ids=get_model_output_tokens(prompt, response),
277 + )
278 +
279 + assert tool_call_tuples(tool_calls) == [
280 + ("math.sum", json.dumps({"a": 2, "b": 3}))
281 + ]
282 +
283 + def test_tool_calls_with_final_content(self, harmony_parser, chat_request):
284 + prompt = [Message.from_role_and_content(Role.USER, "What is the weather?")]
285 + response = [
286 + assistant("User asked about the weather.", "analysis"),
287 + tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),
288 + assistant("This tool call will get the weather.", "final"),
289 + ]
290 +
291 + reasoning, content, tool_calls = harmony_parser.parse(
292 + "",
293 + chat_request,
294 + model_output_token_ids=get_model_output_tokens(prompt, response),
295 + )
296 +
297 + assert reasoning == "User asked about the weather."
298 + assert content == "This tool call will get the weather."
299 + assert tool_call_tuples(tool_calls) == [
300 + ("get_current_weather", json.dumps({"location": "Tokyo"}))
301 + ]
302 +
303 + # Raw/truncated Harmony output streams.
304 +
305 + def test_interrupted_first_message(self, harmony_parser, chat_request):
306 + reasoning, content, tool_calls = harmony_parser.parse(
307 + "",
308 + chat_request,
309 + model_output_token_ids=encode_output(
310 + "<|channel|>final<|message|>I'm in the middle of answering"
311 + ),
312 + )
313 +
314 + assert reasoning is None
315 + assert content == "I'm in the middle of answering"
316 + assert tool_calls is None
317 +
318 + def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request):
319 + reasoning, content, tool_calls = harmony_parser.parse(
320 + "",
321 + chat_request,
322 + model_output_token_ids=encode_output(
323 + "<|channel|>analysis<|message|>I'm in the middle of thinking"
324 + ),
325 + )
326 +
327 + assert reasoning == "I'm in the middle of thinking"
328 + assert content is None
329 + assert tool_calls is None
330 +
331 + def test_truncated_output(self, harmony_parser, chat_request):
332 + reasoning, content, tool_calls = harmony_parser.parse(
333 + "",
334 + chat_request,
335 + model_output_token_ids=encode_output(
336 + "<|channel|>analysis<|message|>I'm thinking.<|end|>"
337 + "<|start|>assistant<|channel|>final<|message|>"
338 + "I'm in the middle of answering"
339 + ),
340 + )
341 +
342 + assert reasoning == "I'm thinking."
343 + assert content == "I'm in the middle of answering"
344 + assert tool_calls is None
345 +
346 + @pytest.mark.parametrize(
347 + ("harmony_str", "expected_content"),
348 + [
349 + (
350 + "<|channel|>commentary<|message|>I'll search for that",
351 + "I'll search for that",
352 + ),
353 + (
354 + "<|channel|>commentary<|message|>Let me look that up.<|end|>"
355 + "<|start|>assistant<|channel|>final<|message|>The answer is 42.<|end|>",
356 + "Let me look that up.\nThe answer is 42.",
357 + ),
358 + ],
359 + )
360 + def test_commentary_preambles(
361 + self,
362 + harmony_parser,
363 + chat_request,
364 + harmony_str,
365 + expected_content,
366 + ):
367 + reasoning, content, tool_calls = harmony_parser.parse(
368 + "",
369 + chat_request,
370 + model_output_token_ids=encode_output(harmony_str),
371 + )
372 +
373 + assert reasoning is None
374 + assert content == expected_content
375 + assert tool_calls is None
376 +
377 + def test_commentary_with_recipient_excluded(self, harmony_parser, chat_request):
378 + reasoning, content, tool_calls = harmony_parser.parse(
379 + "",
380 + chat_request,
381 + model_output_token_ids=encode_output(
382 + "<|channel|>commentary"
383 + "<|message|>Let me check the weather.<|end|>"
384 + "<|start|>assistant to=functions.get_weather"
385 + "<|channel|>commentary"
386 + '<|message|>{"location": "SF"}<|end|>'
387 + ),
388 + )
389 +
390 + assert reasoning is None
391 + assert content == "Let me check the weather."
392 + assert tool_call_tuples(tool_calls) == [
393 + ("get_weather", json.dumps({"location": "SF"}))
394 + ]
395 +
396 +
397 +class TestProcessChunk:
398 + def test_empty(self, harmony_parser):
399 + result = harmony_parser.process_chunk([])
400 + assert result.segments == []
401 + assert result.reasoning_token_count == 0
402 +
403 + def test_single_channel(self, harmony_parser):
404 + result = harmony_parser.process_chunk(
405 + encode_output("<|channel|>final<|message|>Hello")
406 + )
407 +
408 + assert visible_segments(result) == [("final", None, "Hello")]
409 +
410 + def test_cross_channel(self, harmony_parser):
411 + result = harmony_parser.process_chunk(
412 + encode_output(
413 + "<|channel|>analysis<|message|>Think<|end|>"
414 + "<|start|>assistant<|channel|>final<|message|>Answer"
415 + )
416 + )
417 +
418 + assert visible_segments(result) == [
419 + ("analysis", None, "Think"),
420 + ("final", None, "Answer"),
421 + ]
422 +
423 + def test_boundary_detection(self, harmony_parser):
424 + result = harmony_parser.process_chunk(
425 + encode_output("<|channel|>final<|message|>Done<|end|>")
426 + )
427 +
428 + boundary_segments = [
429 + segment for segment in result.segments if segment.is_boundary
430 + ]
431 + assert len(boundary_segments) == 1
432 + assert boundary_segments[0].completed_message is not None
433 + assert boundary_segments[0].completed_message.channel == "final"
434 + assert get_text(boundary_segments[0].completed_message) == "Done"
435 +
436 + def test_multi_boundary(self, harmony_parser):
437 + result = harmony_parser.process_chunk(
438 + encode_output(
439 + "<|channel|>analysis<|message|>One<|end|>"
440 + "<|start|>assistant<|channel|>final<|message|>Two<|end|>"
441 + )
442 + )
443 +
444 + boundary_segments = [
445 + segment for segment in result.segments if segment.is_boundary
446 + ]
447 + assert [
448 + get_text(segment.completed_message) for segment in boundary_segments
449 + ] == [
450 + "One",
451 + "Two",
452 + ]
tests/tool_parsers/test_openai_tool_parser.py deleted
-415
@@ -1,415 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import json
5 -
6 -import pytest
7 -from openai_harmony import (
8 - Conversation,
9 - DeveloperContent,
10 - HarmonyEncodingName,
11 - Message,
12 - Role,
13 - SystemContent,
14 - load_harmony_encoding,
15 -)
16 -
17 -from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
18 -from vllm.tokenizers import get_tokenizer
19 -from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser
20 -
21 -MODEL = "gpt2"
22 -
23 -
24 -@pytest.fixture(scope="module")
25 -def openai_tokenizer():
26 - # The parser does not use the tokenizer, but the constructor requires it.
27 - return get_tokenizer(MODEL)
28 -
29 -
30 -@pytest.fixture
31 -def openai_tool_parser(openai_tokenizer):
32 - return OpenAIToolParser(openai_tokenizer)
33 -
34 -
35 -@pytest.fixture(scope="module")
36 -def harmony_encoding():
37 - return load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
38 -
39 -
40 -def assert_tool_calls(
41 - actual_tool_calls: list[ToolCall],
42 - expected_tool_calls: list[ToolCall],
43 -):
44 - assert len(actual_tool_calls) == len(expected_tool_calls)
45 -
46 - for actual_tool_call, expected_tool_call in zip(
47 - actual_tool_calls, expected_tool_calls
48 - ):
49 - assert isinstance(actual_tool_call.id, str)
50 - assert len(actual_tool_call.id) > 16 # Default from protocol.py
51 - assert actual_tool_call.type == "function"
52 - assert actual_tool_call.function == expected_tool_call.function
53 -
54 -
55 -def test_extract_tool_calls_no_tools(openai_tool_parser, harmony_encoding):
56 - convo = Conversation.from_messages(
57 - [
58 - Message.from_role_and_content(
59 - Role.SYSTEM,
60 - SystemContent.new(),
61 - ),
62 - Message.from_role_and_content(
63 - Role.DEVELOPER,
64 - DeveloperContent.new().with_instructions("Talk like a pirate!"),
65 - ),
66 - Message.from_role_and_content(Role.USER, "Arrr, how be you?"),
67 - Message.from_role_and_content(
68 - Role.ASSISTANT, "This is a test"
69 - ).with_channel("final"),
70 - ]
71 - )
72 - token_ids = harmony_encoding.render_conversation_for_completion(
73 - convo, Role.ASSISTANT
74 - )
75 - extracted_info = openai_tool_parser.extract_tool_calls(
76 - "",
77 - request=None,
78 - token_ids=token_ids,
79 - )
80 - assert not extracted_info.tools_called
81 - assert extracted_info.tool_calls == []
82 - assert extracted_info.content == "This is a test"
83 -
84 -
85 -@pytest.mark.parametrize(
86 - "tool_args",
87 - [
88 - '{"location": "Tokyo"}',
89 - '{\n"location": "Tokyo"\n}',
90 - ],
91 -)
92 -def test_extract_tool_calls_single_tool(
93 - openai_tool_parser, harmony_encoding, tool_args
94 -):
95 - convo = Conversation.from_messages(
96 - [
97 - Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
98 - Message.from_role_and_content(
99 - Role.ASSISTANT,
100 - 'User asks: "What is the weather in Tokyo?" We need to use get_current_weather tool.', # noqa: E501
101 - ).with_channel("analysis"),
102 - Message.from_role_and_content(Role.ASSISTANT, tool_args)
103 - .with_channel("commentary")
104 - .with_recipient("functions.get_current_weather")
105 - .with_content_type("json"),
106 - ]
107 - )
108 - token_ids = harmony_encoding.render_conversation_for_completion(
109 - convo, Role.ASSISTANT
110 - )
111 -
112 - extracted_info = openai_tool_parser.extract_tool_calls(
113 - "",
114 - request=None,
115 - token_ids=token_ids,
116 - )
117 - assert extracted_info.tools_called
118 - expected_tool_calls = [
119 - ToolCall(
120 - function=FunctionCall(
121 - name="get_current_weather",
122 - arguments=json.dumps({"location": "Tokyo"}),
123 - )
124 - )
125 - ]
126 - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
127 - assert extracted_info.content is None
128 -
129 -
130 -def test_extract_tool_calls_multiple_tools(
131 - openai_tool_parser,
132 - harmony_encoding,
133 -):
134 - convo = Conversation.from_messages(
135 - [
136 - Message.from_role_and_content(
137 - Role.USER, "What is the weather in Tokyo based on where I'm at?"
138 - ),
139 - Message.from_role_and_content(
140 - Role.ASSISTANT,
141 - 'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
142 - ).with_channel("analysis"),
143 - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
144 - .with_channel("commentary")
145 - .with_recipient("functions.get_current_weather")
146 - .with_content_type("json"),
147 - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
148 - .with_channel("commentary")
149 - .with_recipient("functions.get_user_location")
150 - .with_content_type("json"),
151 - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
152 - .with_channel("commentary")
153 - .with_recipient("functions.no_content_type"),
154 - Message.from_role_and_content(Role.ASSISTANT, "foo")
155 - .with_channel("commentary")
156 - .with_recipient("functions.not_json_no_content_type"),
157 - Message.from_role_and_content(Role.ASSISTANT, "{}")
158 - .with_channel("commentary")
159 - .with_recipient("functions.empty_args")
160 - .with_content_type("json"),
161 - Message.from_role_and_content(Role.ASSISTANT, "")
162 - .with_channel("commentary")
163 - .with_recipient("functions.no_args")
164 - .with_content_type("json"),
165 - ]
166 - )
167 - token_ids = harmony_encoding.render_conversation_for_completion(
168 - convo,
169 - Role.ASSISTANT,
170 - )
171 -
172 - extracted_info = openai_tool_parser.extract_tool_calls(
173 - "",
174 - request=None,
175 - token_ids=token_ids,
176 - )
177 - assert extracted_info.tools_called
178 - expected_tool_calls = [
179 - ToolCall(
180 - function=FunctionCall(
181 - name="get_current_weather",
182 - arguments=json.dumps({"location": "Tokyo"}),
183 - )
184 - ),
185 - ToolCall(
186 - function=FunctionCall(
187 - name="get_user_location",
188 - arguments=json.dumps({"location": "Tokyo"}),
189 - )
190 - ),
191 - ToolCall(
192 - function=FunctionCall(
193 - name="no_content_type",
194 - arguments=json.dumps({"location": "Tokyo"}),
195 - )
196 - ),
197 - ToolCall(
198 - function=FunctionCall(
199 - name="not_json_no_content_type",
200 - arguments="foo",
201 - )
202 - ),
203 - ToolCall(
204 - function=FunctionCall(
205 - name="empty_args",
206 - arguments=json.dumps({}),
207 - )
208 - ),
209 - ToolCall(
210 - function=FunctionCall(
211 - name="no_args",
212 - arguments="",
213 - )
214 - ),
215 - ]
216 - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
217 - assert extracted_info.content is None
218 -
219 -
220 -def test_extract_tool_calls_bare_function_name(
221 - openai_tool_parser,
222 - harmony_encoding,
223 -):
224 - convo = Conversation.from_messages(
225 - [
226 - Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
227 - Message.from_role_and_content(
228 - Role.ASSISTANT,
229 - "We need to use get_current_weather tool.",
230 - ).with_channel("analysis"),
231 - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
232 - .with_channel("commentary")
233 - .with_recipient("get_current_weather")
234 - .with_content_type("json"),
235 - ]
236 - )
237 - token_ids = harmony_encoding.render_conversation_for_completion(
238 - convo, Role.ASSISTANT
239 - )
240 -
241 - extracted_info = openai_tool_parser.extract_tool_calls(
242 - "",
243 - request=None,
244 - token_ids=token_ids,
245 - )
246 - assert extracted_info.tools_called
247 - expected_tool_calls = [
248 - ToolCall(
249 - function=FunctionCall(
250 - name="get_current_weather",
251 - arguments=json.dumps({"location": "Tokyo"}),
252 - )
253 - )
254 - ]
255 - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
256 - assert extracted_info.content is None
257 -
258 -
259 -def test_extract_tool_calls_bare_function_name_multiple(
260 - openai_tool_parser,
261 - harmony_encoding,
262 -):
263 - convo = Conversation.from_messages(
264 - [
265 - Message.from_role_and_content(
266 - Role.USER, "What is the weather in Tokyo based on where I'm at?"
267 - ),
268 - Message.from_role_and_content(
269 - Role.ASSISTANT,
270 - "We need to use both tools.",
271 - ).with_channel("analysis"),
272 - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
273 - .with_channel("commentary")
274 - .with_recipient("get_current_weather")
275 - .with_content_type("json"),
276 - Message.from_role_and_content(Role.ASSISTANT, "{}")
277 - .with_channel("commentary")
278 - .with_recipient("get_user_location")
279 - .with_content_type("json"),
280 - ]
281 - )
282 - token_ids = harmony_encoding.render_conversation_for_completion(
283 - convo, Role.ASSISTANT
284 - )
285 -
286 - extracted_info = openai_tool_parser.extract_tool_calls(
287 - "",
288 - request=None,
289 - token_ids=token_ids,
290 - )
291 - assert extracted_info.tools_called
292 - expected_tool_calls = [
293 - ToolCall(
294 - function=FunctionCall(
295 - name="get_current_weather",
296 - arguments=json.dumps({"location": "Tokyo"}),
297 - )
298 - ),
299 - ToolCall(
300 - function=FunctionCall(
301 - name="get_user_location",
302 - arguments=json.dumps({}),
303 - )
304 - ),
305 - ]
306 - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
307 -
308 -
309 -def test_extract_tool_calls_assistant_recipient_ignored(
310 - openai_tool_parser,
311 - harmony_encoding,
312 -):
313 - convo = Conversation.from_messages(
314 - [
315 - Message.from_role_and_content(Role.USER, "Hello"),
316 - Message.from_role_and_content(Role.ASSISTANT, "Some tool response")
317 - .with_channel("commentary")
318 - .with_recipient("assistant"),
319 - Message.from_role_and_content(
320 - Role.ASSISTANT, "Here is the answer"
321 - ).with_channel("final"),
322 - ]
323 - )
324 - token_ids = harmony_encoding.render_conversation_for_completion(
325 - convo, Role.ASSISTANT
326 - )
327 -
328 - extracted_info = openai_tool_parser.extract_tool_calls(
329 - "",
330 - request=None,
331 - token_ids=token_ids,
332 - )
333 - assert not extracted_info.tools_called
334 - assert extracted_info.tool_calls == []
335 - assert extracted_info.content == "Here is the answer"
336 -
337 -
338 -def test_extract_tool_calls_dotted_function_name(
339 - openai_tool_parser,
340 - harmony_encoding,
341 -):
342 - convo = Conversation.from_messages(
343 - [
344 - Message.from_role_and_content(Role.USER, "Compute 2+3"),
345 - Message.from_role_and_content(Role.ASSISTANT, '{"a": 2, "b": 3}')
346 - .with_channel("commentary")
347 - .with_recipient("math.sum")
348 - .with_content_type("json"),
349 - ]
350 - )
351 - token_ids = harmony_encoding.render_conversation_for_completion(
352 - convo, Role.ASSISTANT
353 - )
354 -
355 - extracted_info = openai_tool_parser.extract_tool_calls(
356 - "",
357 - request=None,
358 - token_ids=token_ids,
359 - )
360 - assert extracted_info.tools_called
361 - expected_tool_calls = [
362 - ToolCall(
363 - function=FunctionCall(
364 - name="math.sum",
365 - arguments=json.dumps({"a": 2, "b": 3}),
366 - )
367 - )
368 - ]
369 - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
370 -
371 -
372 -def test_extract_tool_calls_with_content(
373 - openai_tool_parser,
374 - harmony_encoding,
375 -):
376 - final_content = "This tool call will get the weather."
377 - convo = Conversation.from_messages(
378 - [
379 - Message.from_role_and_content(
380 - Role.USER, "What is the weather in Tokyo based on where I'm at?"
381 - ),
382 - Message.from_role_and_content(
383 - Role.ASSISTANT,
384 - 'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
385 - ).with_channel("analysis"),
386 - Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
387 - .with_channel("commentary")
388 - .with_recipient("functions.get_current_weather")
389 - .with_content_type("json"),
390 - Message.from_role_and_content(Role.ASSISTANT, final_content).with_channel(
391 - "final"
392 - ),
393 - ]
394 - )
395 - token_ids = harmony_encoding.render_conversation_for_completion(
396 - convo,
397 - Role.ASSISTANT,
398 - )
399 -
400 - extracted_info = openai_tool_parser.extract_tool_calls(
401 - "",
402 - request=None,
403 - token_ids=token_ids,
404 - )
405 - assert extracted_info.tools_called
406 - expected_tool_calls = [
407 - ToolCall(
408 - function=FunctionCall(
409 - name="get_current_weather",
410 - arguments=json.dumps({"location": "Tokyo"}),
411 - )
412 - ),
413 - ]
414 - assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
415 - assert extracted_info.content == final_content
vllm/entrypoints/openai/chat_completion/serving.py
+10 -74
@@ -54,7 +54,6 @@ from vllm.entrypoints.openai.engine.serving import (
54 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
55 from vllm.entrypoints.openai.parser.harmony_utils import (
56 get_streamable_parser_for_assistant,
57 - parse_chat_output,
57 )
58 from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
59 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -135,6 +134,7 @@ class OpenAIServingChat(OpenAIServing):
134 reasoning_parser_name=reasoning_parser,
135 enable_auto_tools=enable_auto_tools,
136 model_name=self.model_config.model,
137 + is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
138 )
139 if (
140 is_mistral_tool_parser(self.tool_parser)
@@ -359,14 +359,6 @@ class OpenAIServingChat(OpenAIServing):
359 assert len(generators) == 1
360 (result_generator,) = generators
361
362 - parser: Parser | None = None
363 - if self.parser_cls is not None:
364 - parser = self.parser_cls(
365 - tokenizer,
366 - request.tools,
367 - chat_template_kwargs=chat_template_kwargs,
368 - )
369 -
362 if request.stream:
363 return self.chat_completion_stream_generator(
364 request,
@@ -387,7 +379,7 @@ class OpenAIServingChat(OpenAIServing):
379 conversation,
380 tokenizer,
381 request_metadata,
390 - parser,
382 + chat_template_kwargs=chat_template_kwargs,
383 )
384
385 def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
@@ -840,7 +832,7 @@ class OpenAIServingChat(OpenAIServing):
832 conversation: list[ConversationMessage],
833 tokenizer: TokenizerLike,
834 request_metadata: RequestResponseMetadata,
843 - parser: Parser | None = None,
835 + chat_template_kwargs: dict[str, Any] | None = None,
836 ) -> ErrorResponse | ChatCompletionResponse:
837 created_time = int(time.time())
838 final_res: RequestOutput | None = None
@@ -871,7 +863,6 @@ class OpenAIServingChat(OpenAIServing):
863 self._raise_if_error(output.finish_reason, request_id)
864 token_ids = output.token_ids
865 out_logprobs = output.logprobs
874 - tool_call_info = None
866
867 if request.logprobs and request.top_logprobs is not None:
868 assert out_logprobs is not None, "Did not output logprobs"
@@ -885,75 +876,20 @@ class OpenAIServingChat(OpenAIServing):
876 else:
877 logprobs = None
878
888 - if self.use_harmony:
889 - reasoning, content, _ = parse_chat_output(token_ids)
890 - if not request.include_reasoning:
891 - reasoning = None
892 -
893 - if self.tool_parser is not None:
894 - if tokenizer is None:
895 - raise ValueError(
896 - "Tokenizer not available when `skip_tokenizer_init=True`"
897 - )
898 -
899 - tool_parser = self.tool_parser(tokenizer, request.tools)
900 - # NOTE: We use token_ids for openai tool parser
901 - tool_call_info = tool_parser.extract_tool_calls(
902 - "",
903 - request=request,
904 - token_ids=token_ids, # type: ignore
905 - )
906 - content = tool_call_info.content
907 - message = ChatMessage(
908 - role=role,
909 - reasoning=reasoning,
910 - content=content,
911 - tool_calls=tool_call_info.tool_calls,
912 - )
913 - else:
914 - message = ChatMessage(
915 - role=role,
916 - reasoning=reasoning,
917 - content=content,
918 - )
919 -
920 - # Encode routed_experts for transport. JSON can't carry raw
921 - # bytes, so we write the ndarray as a ``.npy`` byte stream
922 - # and base64-encode it. ``pybase64`` is ~3x faster than the
923 - # stdlib ``base64`` on large payloads thanks to SIMD.
924 - routed_experts_b64 = None
925 - if output.routed_experts is not None:
926 - buf = io.BytesIO()
927 - np.save(buf, output.routed_experts)
928 - routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
929 - "ascii"
930 - )
931 -
932 - choice_data = ChatCompletionResponseChoice(
933 - index=output.index,
934 - message=message,
935 - logprobs=logprobs,
936 - finish_reason=(
937 - "tool_calls"
938 - if (tool_call_info is not None and tool_call_info.tools_called)
939 - else output.finish_reason
940 - if output.finish_reason
941 - else "stop"
942 - ),
943 - stop_reason=output.stop_reason,
944 - token_ids=(
945 - as_list(output.token_ids) if request.return_token_ids else None
946 - ),
947 - routed_experts=routed_experts_b64,
879 + parser: Parser | None = None
880 + if self.parser_cls is not None:
881 + parser = self.parser_cls(
882 + tokenizer,
883 + request.tools,
884 + chat_template_kwargs=chat_template_kwargs,
885 )
949 - choices.append(choice_data)
950 - continue
886
887 if parser is not None:
888 reasoning, content, tool_calls = parser.parse(
889 output.text,
890 request,
891 enable_auto_tools=self.enable_auto_tools,
892 + model_output_token_ids=token_ids,
893 )
894 if not request.include_reasoning:
895 reasoning = None
vllm/entrypoints/openai/parser/harmony_utils.py
+1 -63
@@ -2,7 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import datetime
5 -from collections.abc import Iterable, Sequence
5 +from collections.abc import Sequence
6 from typing import Any
7
8 from openai.types.responses.tool import Tool
@@ -456,65 +456,3 @@ def render_for_completion(messages: list[Message]) -> list[int]:
456
457 def get_streamable_parser_for_assistant() -> StreamableParser:
458 return StreamableParser(get_encoding(), role=Role.ASSISTANT)
459 -
460 -
461 -def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser:
462 - parser = get_streamable_parser_for_assistant()
463 - for token_id in token_ids:
464 - parser.process(token_id)
465 - return parser
466 -
467 -
468 -def parse_chat_output(
469 - token_ids: Sequence[int],
470 -) -> tuple[str | None, str | None, bool]:
471 - """
472 - Parse the output of a Harmony chat completion into reasoning and final content.
473 - Note that when the `openai` tool parser is used, serving_chat only uses this
474 - for the reasoning content and gets the final content from the tool call parser.
475 -
476 - When the `openai` tool parser is not enabled, or when `GptOssReasoningParser` is
477 - in use,this needs to return the final content without any tool calls parsed.
478 -
479 - Empty reasoning or final content is returned as None instead of an empty string.
480 - """
481 - parser = parse_output_into_messages(token_ids)
482 - output_msgs = parser.messages
483 - is_tool_call = False # TODO: update this when tool call is supported
484 -
485 - # Get completed messages from the parser
486 - # - analysis channel: hidden reasoning
487 - # - commentary channel without recipient (preambles): visible to user
488 - # - final channel: visible to user
489 - # - commentary with recipient (tool calls): handled separately by tool parser
490 - reasoning_texts = [
491 - msg.content[0].text for msg in output_msgs if msg.channel == "analysis"
492 - ]
493 - final_texts = [
494 - msg.content[0].text
495 - for msg in output_msgs
496 - if msg.channel == "final" or (msg.channel == "commentary" and not msg.recipient)
497 - ]
498 -
499 - # Extract partial messages from the parser
500 - if parser.current_channel == "analysis" and parser.current_content:
501 - reasoning_texts.append(parser.current_content)
502 - elif parser.current_channel == "final" and parser.current_content:
503 - final_texts.append(parser.current_content)
504 - elif (
505 - parser.current_channel == "commentary"
506 - and not parser.current_recipient
507 - and parser.current_content
508 - ):
509 - # Preambles (commentary without recipient) are visible to user
510 - final_texts.append(parser.current_content)
511 -
512 - # Flatten multiple messages into a single string
513 - reasoning: str | None = "\n".join(reasoning_texts)
514 - final_content: str | None = "\n".join(final_texts)
515 -
516 - # Return None instead of empty string since existing callers check for None
517 - reasoning = reasoning or None
518 - final_content = final_content or None
519 -
520 - return reasoning, final_content, is_tool_call
vllm/entrypoints/openai/responses/serving.py
+1
@@ -191,6 +191,7 @@ class OpenAIServingResponses(OpenAIServing):
191 reasoning_parser_name=reasoning_parser,
192 enable_auto_tools=enable_auto_tools,
193 model_name=self.model_config.model,
194 + is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
195 )
196 self.enable_prompt_tokens_details = enable_prompt_tokens_details
197 self.enable_force_include_usage = enable_force_include_usage
vllm/parser/__init__.py
+2
@@ -5,10 +5,12 @@ from vllm.parser.abstract_parser import (
5 DelegatingParser,
6 Parser,
7 )
8 +from vllm.parser.harmony import HarmonyParser
9 from vllm.parser.parser_manager import ParserManager
10
11 __all__ = [
12 "Parser",
13 "DelegatingParser",
14 + "HarmonyParser",
15 "ParserManager",
16 ]
vllm/parser/abstract_parser.py
+3
@@ -282,6 +282,7 @@ class Parser:
282 model_output: str,
283 request: ChatCompletionRequest | ResponsesRequest,
284 enable_auto_tools: bool = False,
285 + model_output_token_ids: Sequence[int] = (),
286 ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
287 """Parse a complete model output, extracting reasoning and tool calls.
288
@@ -289,6 +290,7 @@ class Parser:
290 model_output: The complete model-generated string.
291 request: The request object used to generate the output.
292 enable_auto_tools: Whether to enable automatic tool call parsing.
293 + model_output_token_ids: The generated raw output token IDs.
294
295 Returns:
296 A tuple of (reasoning, content, tool_calls).
@@ -642,6 +644,7 @@ class DelegatingParser(Parser):
644 model_output: str,
645 request: ChatCompletionRequest | ResponsesRequest,
646 enable_auto_tools: bool = False,
647 + model_output_token_ids: Sequence[int] = (),
648 ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
649 reasoning, content = self.extract_reasoning(model_output, request)
650 tool_calls, content = self._extract_tool_calls(
vllm/parser/harmony.py new
+240
@@ -0,0 +1,240 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from __future__ import annotations
5 +
6 +import json
7 +from collections.abc import Sequence
8 +from dataclasses import dataclass
9 +from enum import Enum, auto
10 +from typing import TYPE_CHECKING, NamedTuple
11 +
12 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
13 +from vllm.entrypoints.openai.engine.protocol import (
14 + DeltaMessage,
15 + FunctionCall,
16 +)
17 +from vllm.entrypoints.openai.parser.harmony_utils import (
18 + extract_function_from_recipient,
19 + get_streamable_parser_for_assistant,
20 + is_function_recipient,
21 +)
22 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
23 +from vllm.parser.abstract_parser import DelegatingParser
24 +from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
25 +from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
26 +
27 +if TYPE_CHECKING:
28 + from openai_harmony import Message, Role
29 + from openai_harmony import StreamState as HarmonyStreamState
30 +
31 +
32 +class _SegmentType(Enum):
33 + TOOL = auto()
34 + REASONING = auto()
35 + CONTENT = auto()
36 + IGNORE = auto()
37 +
38 + @staticmethod
39 + def from_channel_and_recipient(
40 + channel: str | None, recipient: str | None
41 + ) -> _SegmentType:
42 + if recipient and is_function_recipient(recipient):
43 + return _SegmentType.TOOL
44 + if channel == "analysis":
45 + return _SegmentType.REASONING
46 + if channel == "final" or (channel == "commentary" and recipient is None):
47 + return _SegmentType.CONTENT
48 + return _SegmentType.IGNORE
49 +
50 +
51 +class Segment(NamedTuple):
52 + channel: str | None
53 + recipient: str | None
54 + delta: str
55 + is_boundary: bool = False
56 + completed_message: Message | None = None
57 +
58 +
59 +@dataclass
60 +class ChunkResult:
61 + segments: list[Segment]
62 + reasoning_token_count: int
63 +
64 +
65 +class HarmonyParser(DelegatingParser):
66 + def __init__(self, tokenizer, tools=None, *args, **kwargs):
67 + super().__init__(tokenizer, tools, *args, **kwargs)
68 +
69 + if self._reasoning_parser and not isinstance(
70 + self._reasoning_parser, GptOssReasoningParser
71 + ):
72 + raise ValueError(
73 + "Harmony requires GptOssReasoningParser, "
74 + f"got {self._reasoning_parser.__class__.__name__}."
75 + )
76 +
77 + if self._tool_parser and not isinstance(self._tool_parser, GptOssToolParser):
78 + raise ValueError(
79 + "Harmony requires GptOssToolParser, "
80 + f"got {self._tool_parser.__class__.__name__}."
81 + )
82 +
83 + self._harmony_parser = get_streamable_parser_for_assistant()
84 +
85 + @property
86 + def messages(self) -> list[Message]:
87 + return self._harmony_parser.messages
88 +
89 + @property
90 + def state(self) -> HarmonyStreamState:
91 + return self._harmony_parser.state
92 +
93 + @property
94 + def current_role(self) -> Role | None:
95 + return self._harmony_parser.current_role
96 +
97 + @property
98 + def current_channel(self) -> str | None:
99 + return self._harmony_parser.current_channel
100 +
101 + @property
102 + def current_recipient(self) -> str | None:
103 + return self._harmony_parser.current_recipient
104 +
105 + @property
106 + def current_content(self) -> str:
107 + return self._harmony_parser.current_content
108 +
109 + @property
110 + def current_content_type(self) -> str | None:
111 + return self._harmony_parser.current_content_type
112 +
113 + def parse(
114 + self,
115 + model_output: str,
116 + request: ChatCompletionRequest | ResponsesRequest,
117 + enable_auto_tools: bool = False,
118 + model_output_token_ids: Sequence[int] = (),
119 + ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
120 + """Parse Harmony output from token IDs.
121 +
122 + Tool calls are always extracted regardless of ``enable_auto_tools``.
123 + Callers must decide whether to surface them.
124 + """
125 + result = self.process_chunk(model_output_token_ids)
126 +
127 + reasoning_parts: list[str] = []
128 + content_parts: list[str] = []
129 + tool_calls: list[FunctionCall] = []
130 +
131 + def _append_parsed_message(
132 + channel: str | None,
133 + recipient: str | None,
134 + text: str,
135 + content_type: str | None = None,
136 + ) -> None:
137 + segment_type = _SegmentType.from_channel_and_recipient(channel, recipient)
138 + match segment_type:
139 + case _SegmentType.REASONING if self.reasoning_parser and text:
140 + reasoning_parts.append(text)
141 + case _SegmentType.CONTENT if text:
142 + content_parts.append(text)
143 + case _SegmentType.TOOL if self.tool_parser:
144 + assert recipient is not None
145 + if content_type is not None and "json" not in content_type:
146 + arguments = text
147 + else:
148 + try:
149 + arguments = json.dumps(json.loads(text))
150 + except json.JSONDecodeError:
151 + arguments = text
152 + tool_calls.append(
153 + FunctionCall(
154 + name=extract_function_from_recipient(recipient),
155 + arguments=arguments,
156 + )
157 + )
158 +
159 + for segment in result.segments:
160 + msg = segment.completed_message
161 + if msg is None:
162 + continue
163 + if msg.author.role != "assistant" or not msg.content:
164 + continue
165 + _append_parsed_message(
166 + channel=msg.channel,
167 + recipient=msg.recipient,
168 + text=msg.content[0].text,
169 + content_type=msg.content_type,
170 + )
171 +
172 + if (
173 + self.current_channel is not None
174 + or self.current_recipient is not None
175 + or self.current_content
176 + ):
177 + _append_parsed_message(
178 + channel=self.current_channel,
179 + recipient=self.current_recipient,
180 + text=self.current_content,
181 + content_type=self.current_content_type,
182 + )
183 +
184 + reasoning = "\n".join(reasoning_parts) or None
185 + content = "\n".join(content_parts) or None
186 + return reasoning, content, tool_calls or None
187 +
188 + def parse_delta(
189 + self,
190 + delta_text: str,
191 + delta_token_ids: list[int],
192 + request: ChatCompletionRequest | ResponsesRequest,
193 + prompt_token_ids: list[int] | None = None,
194 + *,
195 + finished: bool,
196 + ) -> DeltaMessage | None:
197 + raise NotImplementedError(
198 + "HarmonyParser streaming parsing is deferred. "
199 + "Use the existing harmony streaming path."
200 + )
201 +
202 + def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult:
203 + if not token_ids:
204 + return ChunkResult(segments=[], reasoning_token_count=0)
205 +
206 + from openai_harmony import StreamState
207 +
208 + segments: list[Segment] = []
209 + reasoning_token_count = 0
210 + for token_id in token_ids:
211 + self._harmony_parser.process(token_id)
212 + channel = self.current_channel
213 + recipient = self.current_recipient
214 + delta = self._harmony_parser.last_content_delta or ""
215 + completed_message = None
216 + is_boundary = self.state == StreamState.EXPECT_START
217 + if is_boundary and self.messages:
218 + completed_message = self.messages[-1]
219 +
220 + if channel == "analysis" or (
221 + channel == "commentary" and recipient is not None
222 + ):
223 + reasoning_token_count += 1
224 +
225 + segments.append(
226 + Segment(
227 + channel=channel,
228 + recipient=recipient,
229 + delta=delta,
230 + is_boundary=is_boundary,
231 + completed_message=completed_message,
232 + )
233 + )
234 +
235 + # TODO: Optionally merge and suppress empty Segments
236 +
237 + return ChunkResult(
238 + segments=segments,
239 + reasoning_token_count=reasoning_token_count,
240 + )
vllm/parser/mistral.py
+6 -1
@@ -3,6 +3,7 @@
3
4 from __future__ import annotations
5
6 +from collections.abc import Sequence
7 from typing import TYPE_CHECKING
8
9 from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
@@ -43,10 +44,14 @@ class MistralParser(DelegatingParser):
44 model_output: str,
45 request: ChatCompletionRequest | ResponsesRequest,
46 enable_auto_tools: bool = False,
47 + model_output_token_ids: Sequence[int] = (),
48 ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
49 self._maybe_force_auto_tool_parsing(request)
50 reasoning, content, tool_calls = super().parse(
49 - model_output, request, enable_auto_tools
51 + model_output,
52 + request,
53 + enable_auto_tools,
54 + model_output_token_ids,
55 )
56 if tool_calls:
57 from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
vllm/parser/parser_manager.py
+10
@@ -79,6 +79,7 @@ class ParserManager:
79 reasoning_parser_name: str | None = None,
80 enable_auto_tools: bool = False,
81 model_name: str | None = None,
82 + is_harmony: bool = False,
83 ) -> type[Parser] | None:
84 """
85 Get a Parser that handles both reasoning and tool parsing.
@@ -91,6 +92,8 @@ class ParserManager:
92 reasoning_parser_name: The name of the reasoning parser.
93 enable_auto_tools: Whether auto tool choice is enabled.
94 model_name: The model name for parser-specific warnings.
95 + is_harmony: Whether the selected model uses the Harmony format.
96 + If True, HarmonyParser is always returned.
97
98 Returns:
99 A Parser class, or None if neither parser is specified.
@@ -108,6 +111,13 @@ class ParserManager:
111
112 from vllm.utils.mistral import is_mistral_tool_parser
113
114 + if is_harmony:
115 + from vllm.parser.harmony import HarmonyParser
116 +
117 + HarmonyParser.reasoning_parser_cls = reasoning_parser_cls
118 + HarmonyParser.tool_parser_cls = tool_parser_cls
119 + return HarmonyParser
120 +
121 if is_mistral_tool_parser(tool_parser_cls):
122 from vllm.parser.mistral import MistralParser
123
vllm/reasoning/gptoss_reasoning_parser.py
+10 -25
@@ -8,7 +8,6 @@ from transformers import PreTrainedTokenizerBase
8
9 from vllm.entrypoints.mcp.tool_server import ToolServer
10 from vllm.entrypoints.openai.engine.protocol import DeltaMessage
11 -from vllm.entrypoints.openai.parser.harmony_utils import parse_chat_output
11 from vllm.logger import init_logger
12 from vllm.reasoning import ReasoningParser
13
@@ -132,10 +131,10 @@ class GptOssReasoningParser(ReasoningParser):
131 return self.is_reasoning_end(input_ids[n - window :])
132
133 def extract_content_ids(self, input_ids: list[int]) -> list[int]:
135 - _, content, _ = parse_chat_output(input_ids)
136 - if content is None:
137 - return []
138 - return self.model_tokenizer.encode(content)
134 + raise NotImplementedError(
135 + "GptOssReasoningParser only provides boundary detection. "
136 + "Use HarmonyParser for output parsing."
137 + )
138
139 def extract_reasoning_streaming(
140 self,
@@ -146,25 +145,10 @@ class GptOssReasoningParser(ReasoningParser):
145 current_token_ids: Sequence[int],
146 delta_token_ids: Sequence[int],
147 ) -> DeltaMessage | None:
149 - prev_reasoning, prev_content, _ = parse_chat_output(list(previous_token_ids))
150 - cur_reasoning, cur_content, _ = parse_chat_output(list(current_token_ids))
151 - reasoning_delta = None
152 - content_delta = None
153 - if cur_reasoning is not None:
154 - prev_r = prev_reasoning or ""
155 - if cur_reasoning.startswith(prev_r):
156 - reasoning_delta = cur_reasoning[len(prev_r) :] or None
157 - else:
158 - reasoning_delta = cur_reasoning
159 - if cur_content is not None:
160 - prev_c = prev_content or ""
161 - if cur_content.startswith(prev_c):
162 - content_delta = cur_content[len(prev_c) :] or None
163 - else:
164 - content_delta = cur_content
165 - if reasoning_delta is None and content_delta is None:
166 - return None
167 - return DeltaMessage(reasoning=reasoning_delta, content=content_delta)
148 + raise NotImplementedError(
149 + "GptOssReasoningParser only provides boundary detection. "
150 + "Use HarmonyParser for output parsing."
151 + )
152
153 def extract_reasoning(
154 self,
@@ -172,7 +156,8 @@ class GptOssReasoningParser(ReasoningParser):
156 request: "ChatCompletionRequest | ResponsesRequest",
157 ) -> tuple[str | None, str | None]:
158 raise NotImplementedError(
175 - "gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501
159 + "GptOssReasoningParser only provides boundary detection. "
160 + "Use HarmonyParser for output parsing."
161 )
162
163 # This function prepares the structural tag to format reasoning output
vllm/tool_parsers/__init__.py
+2 -2
@@ -143,8 +143,8 @@ _TOOL_PARSERS_TO_REGISTER = {
143 "Olmo3PythonicToolParser",
144 ),
145 "openai": (
146 - "openai_tool_parser",
147 - "OpenAIToolParser",
146 + "gptoss_tool_parser",
147 + "GptOssToolParser",
148 ),
149 "phi4_mini_json": (
150 "phi4mini_tool_parser",
vllm/tool_parsers/gptoss_tool_parser.py new
+47
@@ -0,0 +1,47 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from collections.abc import Sequence
5 +from typing import TYPE_CHECKING
6 +
7 +from vllm.entrypoints.openai.engine.protocol import (
8 + DeltaMessage,
9 + ExtractedToolCallInformation,
10 +)
11 +from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
12 +
13 +if TYPE_CHECKING:
14 + from vllm.tokenizers import TokenizerLike
15 +
16 +
17 +class GptOssToolParser(ToolParser):
18 + """
19 + Stub tool parser for gpt-oss/harmony models.
20 +
21 + All output parsing is handled by HarmonyParser. This stub exists as a
22 + capability declaration via HarmonyParser.tool_parser_cls.
23 + """
24 +
25 + def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
26 + super().__init__(tokenizer, tools)
27 +
28 + def extract_tool_calls(
29 + self, model_output, request, **kwargs
30 + ) -> ExtractedToolCallInformation:
31 + raise NotImplementedError(
32 + "GptOssToolParser is a stub. Use HarmonyParser for tool parsing."
33 + )
34 +
35 + def extract_tool_calls_streaming(
36 + self,
37 + previous_text: str,
38 + current_text: str,
39 + delta_text: str,
40 + previous_token_ids: Sequence[int],
41 + current_token_ids: Sequence[int],
42 + delta_token_ids: Sequence[int],
43 + request,
44 + ) -> DeltaMessage | None:
45 + raise NotImplementedError(
46 + "GptOssToolParser is a stub. Use HarmonyParser for tool parsing."
47 + )
vllm/tool_parsers/openai_tool_parser.py deleted
-120
@@ -1,120 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import json
4 -from collections.abc import Sequence
5 -from typing import TYPE_CHECKING
6 -
7 -from vllm.entrypoints.openai.chat_completion.protocol import (
8 - ChatCompletionRequest,
9 -)
10 -from vllm.entrypoints.openai.engine.protocol import (
11 - DeltaMessage,
12 - ExtractedToolCallInformation,
13 - FunctionCall,
14 - ToolCall,
15 -)
16 -from vllm.entrypoints.openai.parser.harmony_utils import (
17 - extract_function_from_recipient,
18 - is_function_recipient,
19 - parse_output_into_messages,
20 -)
21 -from vllm.logger import init_logger
22 -from vllm.tool_parsers.abstract_tool_parser import (
23 - Tool,
24 - ToolParser,
25 -)
26 -
27 -if TYPE_CHECKING:
28 - from vllm.tokenizers import TokenizerLike
29 -else:
30 - TokenizerLike = object
31 -
32 -logger = init_logger(__name__)
33 -
34 -
35 -class OpenAIToolParser(ToolParser):
36 - def __init__(self, tokenizer: "TokenizerLike", tools: list[Tool] | None = None):
37 - super().__init__(tokenizer, tools)
38 -
39 - def extract_tool_calls(
40 - self,
41 - model_output: str,
42 - request: ChatCompletionRequest,
43 - token_ids: Sequence[int] | None = None,
44 - ) -> ExtractedToolCallInformation:
45 - if token_ids is None:
46 - raise NotImplementedError(
47 - "OpenAIToolParser requires token IDs and does not support text-based extraction." # noqa: E501
48 - )
49 -
50 - parser = parse_output_into_messages(token_ids)
51 - tool_calls = []
52 - final_content = None
53 - commentary_content = None
54 -
55 - if len(parser.messages) > 0:
56 - for msg in parser.messages:
57 - if msg.author.role != "assistant":
58 - continue
59 - if len(msg.content) < 1:
60 - continue
61 - msg_text = msg.content[0].text
62 - if msg.recipient and is_function_recipient(msg.recipient):
63 - # If no content-type is given assume JSON, as that's the
64 - # most common case with gpt-oss models.
65 - if not msg.content_type or "json" in msg.content_type:
66 - # load and dump the JSON text to check validity and
67 - # remove any extra newlines or other odd formatting
68 - try:
69 - tool_args = json.dumps(json.loads(msg_text))
70 - except json.JSONDecodeError:
71 - logger.exception(
72 - "Error decoding JSON tool call from response."
73 - )
74 - tool_args = msg_text
75 - else:
76 - tool_args = msg_text
77 - tool_calls.append(
78 - ToolCall(
79 - type="function",
80 - function=FunctionCall(
81 - name=extract_function_from_recipient(msg.recipient),
82 - arguments=tool_args,
83 - ),
84 - )
85 - )
86 - elif msg.channel == "final":
87 - final_content = msg_text
88 - elif msg.channel == "commentary" and not msg.recipient:
89 - commentary_content = msg_text
90 -
91 - # Extract partial content from the parser state if the generation was truncated
92 - if parser.current_content:
93 - if parser.current_channel == "final":
94 - final_content = parser.current_content
95 - elif (
96 - parser.current_channel == "commentary" and not parser.current_recipient
97 - ):
98 - commentary_content = parser.current_content
99 -
100 - return ExtractedToolCallInformation(
101 - tools_called=len(tool_calls) > 0,
102 - tool_calls=tool_calls,
103 - # prefer final content over commentary content if both are present
104 - # commentary content is tool call preambles meant to be shown to the user
105 - content=final_content or commentary_content,
106 - )
107 -
108 - def extract_tool_calls_streaming(
109 - self,
110 - previous_text: str,
111 - current_text: str,
112 - delta_text: str,
113 - previous_token_ids: Sequence[int],
114 - current_token_ids: Sequence[int],
115 - delta_token_ids: Sequence[int],
116 - request: ChatCompletionRequest,
117 - ) -> DeltaMessage | None:
118 - raise NotImplementedError(
119 - "Not being used, manual parsing in serving_chat.py" # noqa: E501
120 - )