@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 51d1b45182

[Bugfix][GPT-OSS] Fix strict tool-call grammar to accept Harmony renders (#52222)

Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>

yzong-rh committed Aug 26, 2026 at 21:05 UTC 51d1b45182a1e9273ef19d2cbfb62055d3aee20f

2 files changed +111 -81

tests/parser/test_harmony.py

+87 -61

@@ -89,8 +89,11 @@ def encode_output(harmony_str: str) -> list[int]:

89 return get_encoding().encode(harmony_str, allowed_special="all")

92 -def assistant(content: str, channel: str) -> Message:

93 - return Message.from_role_and_content(Role.ASSISTANT, content).with_channel(channel)

92 +def assistant(content: str, channel: str, content_type: str | None = None) -> Message:

93 + message = Message.from_role_and_content(Role.ASSISTANT, content).with_channel(

94 + channel

95 + )

96 + return message if content_type is None else message.with_content_type(content_type)

99 def tool_call(

@@ -103,16 +106,14 @@ def tool_call(

106 return message if content_type is None else message.with_content_type(content_type)

107

108

106 -def get_model_output_tokens(

107 - prompt_messages: Sequence[Message],

108 - response_messages: Sequence[Message],

109 -) -> list[int]:

109 +def get_model_output_tokens(response_messages: Sequence[Message]) -> list[int]:

110 enc = get_encoding()

111 + prompt_messages = [Message.from_role_and_content(Role.USER, "x")]

112 # Keep analysis messages when synthesizing model-output-only token sequences

113 # for parser tests; the default render path drops them after a later final turn.

114 config = RenderConversationConfig(auto_drop_analysis=False)

115 prompt_ids = enc.render_conversation_for_completion(

115 - Conversation.from_messages(list(prompt_messages)),

116 + Conversation.from_messages(prompt_messages),

117 Role.ASSISTANT,

118 config=config,

119 )

@@ -124,6 +125,10 @@ def get_model_output_tokens(

125 return full_ids[len(prompt_ids) :]

126

127

128 +def get_model_output_str(response_messages: Sequence[Message]) -> str:

129 + return get_encoding().decode_utf8(get_model_output_tokens(response_messages))

130 +

131 +

132 def get_text(msg: Message) -> str:

133 return msg.content[0].text if msg.content else ""

134

@@ -215,13 +220,12 @@ class TestParse:

220 # Rendered conversation outputs.

221

222 def test_reasoning_only(self, harmony_parser, chat_request):

218 - prompt = [Message.from_role_and_content(Role.USER, "Why?")]

223 response = [assistant("This is reasoning", "analysis")]

224

225 reasoning, content, tool_calls = harmony_parser.parse(

226 "",

227 chat_request,

224 - model_output_token_ids=get_model_output_tokens(prompt, response),

228 + model_output_token_ids=get_model_output_tokens(response),

229 )

230

231 assert reasoning == "This is reasoning"

@@ -229,13 +233,12 @@ class TestParse:

233 assert tool_calls is None

234

235 def test_content_only(self, harmony_parser, chat_request):

232 - prompt = [Message.from_role_and_content(Role.USER, "Hello")]

236 response = [assistant("This is a test", "final")]

237

238 reasoning, content, tool_calls = harmony_parser.parse(

239 "",

240 chat_request,

238 - model_output_token_ids=get_model_output_tokens(prompt, response),

241 + model_output_token_ids=get_model_output_tokens(response),

242 )

243

244 assert reasoning is None

@@ -243,7 +246,6 @@ class TestParse:

246 assert tool_calls is None

247

248 def test_reasoning_and_content(self, harmony_parser, chat_request):

246 - prompt = [Message.from_role_and_content(Role.USER, "What is 2+2?")]

249 response = [

250 assistant("I should think first.", "analysis"),

251 assistant("The answer is 4.", "final"),

@@ -252,7 +254,7 @@ class TestParse:

254 reasoning, content, tool_calls = harmony_parser.parse(

255 "",

256 chat_request,

255 - model_output_token_ids=get_model_output_tokens(prompt, response),

257 + model_output_token_ids=get_model_output_tokens(response),

258 )

259

260 assert reasoning == "I should think first."

@@ -270,15 +272,12 @@ class TestParse:

272 def test_single_tool_call(

273 self, harmony_parser, chat_request, tool_args, tool_channel

274 ):

273 - prompt = [

274 - Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?")

275 - ]

275 response = [tool_call("functions.get_current_weather", tool_args, tool_channel)]

276

277 reasoning, content, tool_calls = harmony_parser.parse(

278 "",

279 chat_request,

281 - model_output_token_ids=get_model_output_tokens(prompt, response),

280 + model_output_token_ids=get_model_output_tokens(response),

281 )

282

283 assert reasoning is None

@@ -288,11 +287,6 @@ class TestParse:

287 ]

288

289 def test_multiple_tool_calls_varied_formats(self, harmony_parser, chat_request):

291 - prompt = [

292 - Message.from_role_and_content(

293 - Role.USER, "What is the weather in Tokyo based on where I'm at?"

294 - )

295 - ]

290 response = [

291 tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),

292 tool_call("functions.get_user_location", '{"location": "Tokyo"}'),

@@ -309,7 +303,7 @@ class TestParse:

303 _, content, tool_calls = harmony_parser.parse(

304 "",

305 chat_request,

312 - model_output_token_ids=get_model_output_tokens(prompt, response),

306 + model_output_token_ids=get_model_output_tokens(response),

307 )

308

309 assert content is None

@@ -323,13 +317,12 @@ class TestParse:

317 ]

318

319 def test_tool_call_bare_recipient(self, harmony_parser, chat_request):

326 - prompt = [Message.from_role_and_content(Role.USER, "Weather?")]

320 response = [tool_call("get_current_weather", '{"location": "Tokyo"}')]

321

322 _, _, tool_calls = harmony_parser.parse(

323 "",

324 chat_request,

332 - model_output_token_ids=get_model_output_tokens(prompt, response),

325 + model_output_token_ids=get_model_output_tokens(response),

326 )

327

328 assert tool_call_tuples(tool_calls) == [

@@ -337,7 +330,6 @@ class TestParse:

330 ]

331

332 def test_multiple_tool_calls_bare_recipients(self, harmony_parser, chat_request):

340 - prompt = [Message.from_role_and_content(Role.USER, "Use both tools.")]

333 response = [

334 tool_call("get_current_weather", '{"location": "Tokyo"}'),

335 tool_call("get_user_location", "{}"),

@@ -346,7 +338,7 @@ class TestParse:

338 _, _, tool_calls = harmony_parser.parse(

339 "",

340 chat_request,

349 - model_output_token_ids=get_model_output_tokens(prompt, response),

341 + model_output_token_ids=get_model_output_tokens(response),

342 )

343

344 assert tool_call_tuples(tool_calls) == [

@@ -355,7 +347,6 @@ class TestParse:

347 ]

348

349 def test_assistant_recipient_not_tool(self, harmony_parser, chat_request):

358 - prompt = [Message.from_role_and_content(Role.USER, "Hello")]

350 response = [

351 tool_call("assistant", "Some tool response", content_type=None),

352 assistant("Here is the answer", "final"),

@@ -364,7 +355,7 @@ class TestParse:

355 reasoning, content, tool_calls = harmony_parser.parse(

356 "",

357 chat_request,

367 - model_output_token_ids=get_model_output_tokens(prompt, response),

358 + model_output_token_ids=get_model_output_tokens(response),

359 )

360

361 assert reasoning is None

@@ -372,13 +363,12 @@ class TestParse:

363 assert tool_calls is None

364

365 def test_tool_call_dotted_name(self, harmony_parser, chat_request):

375 - prompt = [Message.from_role_and_content(Role.USER, "Compute 2+3")]

366 response = [tool_call("math.sum", '{"a": 2, "b": 3}')]

367

368 _, _, tool_calls = harmony_parser.parse(

369 "",

370 chat_request,

381 - model_output_token_ids=get_model_output_tokens(prompt, response),

371 + model_output_token_ids=get_model_output_tokens(response),

372 )

373

374 assert tool_call_tuples(tool_calls) == [

@@ -386,7 +376,6 @@ class TestParse:

376 ]

377

378 def test_tool_calls_with_final_content(self, harmony_parser, chat_request):

389 - prompt = [Message.from_role_and_content(Role.USER, "What is the weather?")]

379 response = [

380 assistant("User asked about the weather.", "analysis"),

381 tool_call("functions.get_current_weather", '{"location": "Tokyo"}'),

@@ -396,7 +385,7 @@ class TestParse:

385 reasoning, content, tool_calls = harmony_parser.parse(

386 "",

387 chat_request,

399 - model_output_token_ids=get_model_output_tokens(prompt, response),

388 + model_output_token_ids=get_model_output_tokens(response),

389 )

390

391 assert reasoning == "User asked about the weather."

@@ -700,12 +689,11 @@ class TestParseDelta:

689 recipient,

690 ):

691 parser = HarmonyParser(gpt_oss_tokenizer)

703 - prompt = [Message.from_role_and_content(Role.USER, "Hello")]

692 response = [tool_call(recipient, "Ignore this", content_type=None)]

693

694 delta = parser.parse_delta(

695 delta_text="",

708 - delta_token_ids=get_model_output_tokens(prompt, response),

696 + delta_token_ids=get_model_output_tokens(response),

697 request=chat_request,

698 finished=False,

699 )

@@ -914,35 +902,61 @@ class TestAdjustRequest:

902 "required": ["answer"],

903 }

904

918 - COMMENTARY = (

905 + ANALYSIS = get_model_output_str([assistant("reasoning", "analysis")])

906 + COMMENTARY = get_model_output_str([assistant("commentary", "commentary")])

907 + TOOL_CALL_1_CHANNEL_FIRST = (

908 + ANALYSIS + f"<|start|>assistant<|channel|>commentary to=functions.{TOOL_1_NAME}"

909 + " <|constrain|>json<|message|>{}<|call|>"

910 )

921 - TOOL_CALL_1 = (

922 - ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_1_NAME} json<|message|>"

923 - "{}<|call|>"

911 + TOOL_CALL_1_FUNCTION_FIRST = get_model_output_str(

912 + [

913 + assistant("reasoning", "analysis"),

914 + tool_call(f"functions.{TOOL_1_NAME}", "{}"),

915 + ]

916 )

925 - TOOL_CALL_2 = (

926 - ANALYSIS + f"<|channel|>commentary to=functions.{TOOL_2_NAME} json<|message|>"

927 - '{"city": "Tokyo"}<|call|>'

917 + TOOL_CALL_2_CHANNEL_FIRST = (

918 + ANALYSIS + f"<|start|>assistant<|channel|>commentary to=functions.{TOOL_2_NAME}"

919 + ' <|constrain|>json<|message|>{"city": "Tokyo"}<|call|>'

920 )

929 - FINAL_JSON_SCHEMA = (

930 - ANALYSIS

931 - + '<|channel|>final <|constrain|>json<|message|>{"answer": "Tokyo"}<|end|>'

921 + TOOL_CALL_2_FUNCTION_FIRST = get_model_output_str(

922 + [

923 + assistant("reasoning", "analysis"),

924 + tool_call(f"functions.{TOOL_2_NAME}", '{"city": "Tokyo"}'),

925 + ]

926 )

933 - FINAL_JSON_OBJECT = (

934 - ANALYSIS

935 - + '<|channel|>final <|constrain|>json<|message|>{"city": "Tokyo"}<|end|>'

927 + FINAL_JSON_SCHEMA = get_model_output_str(

928 + [

929 + assistant("reasoning", "analysis"),

930 + assistant('{"answer": "Tokyo"}', "final", "json"),

931 + ]

932 + )

933 + FINAL_JSON_OBJECT = get_model_output_str(

934 + [

935 + assistant("reasoning", "analysis"),

936 + assistant('{"city": "Tokyo"}', "final", "<|constrain|>json"),

937 + ]

938 + )

939 + FINAL_TEXT_ONLY = get_model_output_str(

940 + [assistant("reasoning", "analysis"), assistant("any", "final")]

941 + )

942 + FINAL_REGEX = get_model_output_str(

943 + [assistant("reasoning", "analysis"), assistant("regex", "final")]

944 + )

945 + FINAL_CHOICE = get_model_output_str(

946 + [assistant("reasoning", "analysis"), assistant("choice1", "final")]

947 + )

948 + FINAL_GRAMMAR = get_model_output_str(

949 + [assistant("reasoning", "analysis"), assistant("grammar", "final")]

950 + )

951 + FINAL_STRUCTURAL_TAG = get_model_output_str(

952 + [assistant("reasoning", "analysis"), assistant("tag content", "final")]

953 )

937 - FINAL_TEXT_ONLY = ANALYSIS + "<|channel|>final<|message|>any<|end|>"

954 ADMISSION_SAMPLES = (

955 "COMMENTARY",

944 - "TOOL_CALL_1",

945 - "TOOL_CALL_2",

956 + "TOOL_CALL_1_CHANNEL_FIRST",

957 + "TOOL_CALL_1_FUNCTION_FIRST",

958 + "TOOL_CALL_2_CHANNEL_FIRST",

959 + "TOOL_CALL_2_FUNCTION_FIRST",

960 "FINAL_JSON_SCHEMA",

961 "FINAL_JSON_OBJECT",

962 "FINAL_TEXT_ONLY",

@@ -1090,8 +1104,10 @@ class TestAdjustRequest:

1104 {"tool_choice": "auto", "strict_tools": True},

1105 [

1106 "COMMENTARY",

1093 - "TOOL_CALL_1",

1094 - "TOOL_CALL_2",

1107 + "TOOL_CALL_1_CHANNEL_FIRST",

1108 + "TOOL_CALL_1_FUNCTION_FIRST",

1109 + "TOOL_CALL_2_CHANNEL_FIRST",

1110 + "TOOL_CALL_2_FUNCTION_FIRST",

1111 "FINAL_JSON_SCHEMA",

1112 "FINAL_JSON_OBJECT",

1113 "FINAL_TEXT_ONLY",

@@ -1103,11 +1119,21 @@ class TestAdjustRequest:

1119 ),

1120 (

1121 {"tool_choice": "required"},

1106 - ["COMMENTARY", "TOOL_CALL_1", "TOOL_CALL_2"],

1122 + [

1123 + "COMMENTARY",

1124 + "TOOL_CALL_1_CHANNEL_FIRST",

1125 + "TOOL_CALL_1_FUNCTION_FIRST",

1126 + "TOOL_CALL_2_CHANNEL_FIRST",

1127 + "TOOL_CALL_2_FUNCTION_FIRST",

1128 + ],

1129 ),

1130 (

1131 {"tool_choice": "named"},

1110 - ["COMMENTARY", "TOOL_CALL_2"],

1132 + [

1133 + "COMMENTARY",

1134 + "TOOL_CALL_2_CHANNEL_FIRST",

1135 + "TOOL_CALL_2_FUNCTION_FIRST",

1136 + ],

1137 ),

1138 (

1139 {"response_format_type": "json_schema"},

vllm/parser/harmony.py

+24 -20

@@ -387,6 +387,7 @@ class HarmonyParser(DelegatingParser):

387 return recipient[:constrain_index].rstrip() or None

388

389

390 +_JSON_CONSTRAINS = [" json", " <|constrain|>json"]

391 # Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|>

392 # They are represented as "" since xgrammar disallows stop tokens while

393 # under constraints, leading to bad or infinite generations.

@@ -398,10 +399,8 @@ _TOOL_CALL_CHANNELS = [

399 "<|channel|>final",

400 ]

401 _FUNCTION_CALL_BEGINS = [

401 - "to=functions.{name} {channel} json<|message|>",

402 - "to=functions.{name} {channel} <|constrain|>json<|message|>",

403 - "{channel} to=functions.{name} json<|message|>",

404 - "{channel} to=functions.{name} <|constrain|>json<|message|>",

402 + " to=functions.{name}{channel}{constrain}<|message|>",

403 + "{channel} to=functions.{name}{constrain}<|message|>",

404 ]

405 _JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"})

406 _ANY_CONTENT = AnyTextFormat()

@@ -476,7 +475,9 @@ def get_harmony_structural_tag(

475 else:

476 tags = [

477 TagFormat(

479 - begin=pattern.format(name=tool.function.name, channel=channel),

478 + begin=pattern.format(

479 + name=tool.function.name, channel=channel, constrain=constrain

480 + ),

481 content=JSONSchemaFormat(

482 json_schema=get_function_parameters(tool.function)

483 ),

@@ -484,23 +485,18 @@ def get_harmony_structural_tag(

485 )

486 for tool in tools

487 for pattern in _FUNCTION_CALL_BEGINS

488 + for constrain in _JSON_CONSTRAINS

489 for channel in _TOOL_CALL_CHANNELS

490 ]

491

492 if tool_choice == "auto":

491 - tags.append(

492 - TagFormat(

493 - begin=_FINAL_BEGIN.format(constrain=" <|constrain|>json"),

494 - content=_ANY_CONTENT,

495 - end=_END_TAG,

496 - )

497 - )

498 - tags.append(

493 + tags.extend(

494 TagFormat(

500 - begin=_FINAL_BEGIN.format(constrain=""),

495 + begin=_FINAL_BEGIN.format(constrain=constrain),

496 content=_ANY_CONTENT,

497 end=_END_TAG,

498 )

499 + for constrain in _JSON_CONSTRAINS + [""]

500 )

501

502 return _assemble_tag(

@@ -558,14 +554,22 @@ def _adjust_output_format(

554 return request

555

556 if isinstance(final_content, JSONSchemaFormat):

561 - begin = _FINAL_BEGIN.format(constrain=" <|constrain|>json")

557 + content = OrFormat(

558 + elements=[

559 + TagFormat(

560 + begin=_FINAL_BEGIN.format(constrain=constrain),

561 + content=final_content,

562 + end=_END_TAG,

563 + )

564 + for constrain in _JSON_CONSTRAINS

565 + ]

566 + )

567 else:

563 - begin = _FINAL_BEGIN.format(constrain="")

564 -

568 + content = TagFormat(

569 + begin=_FINAL_BEGIN.format(constrain=""), content=final_content, end=_END_TAG

570 + )

571 structural_tag = _assemble_tag(

566 - allow_analysis=True,

567 - allow_commentary=False,

568 - content=TagFormat(begin=begin, content=final_content, end=_END_TAG),

572 + allow_analysis=True, allow_commentary=False, content=content

573 )

574

575 request.structured_outputs = replace(