A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 04:15 UTC
[Frontend] [gpt-oss] Tool json call parsing error retry (#27675)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com> Signed-off-by: Alec Solder <alecs@fb.com> Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com> Co-authored-by: Jialin Ouyang <Jialin.Ouyang@gmail.com> Co-authored-by: Alec Solder <alecs@fb.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com>
Alec S committed
Oct 29, 2025 at 05:42 UTC
3c7fefdeba183e5c5e575f668b797549530f5a3d
3 files changed
+62
-3
vllm/entrypoints/context.py
+37
-2
@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Union
11
from openai.types.responses.tool import Mcp
12
from openai_harmony import Author, Message, Role, StreamState, TextContent
13
14
+from vllm import envs
15
from vllm.entrypoints.harmony_utils import (
16
get_encoding,
17
get_streamable_parser_for_assistant,
@@ -109,6 +110,28 @@ class ConversationContext(ABC):
110
raise NotImplementedError("Should not be called.")
111
112
113
+def _create_json_parse_error_messages(
114
+ last_msg: Message, e: json.JSONDecodeError
115
+) -> list[Message]:
116
+ """
117
+ Creates an error message when json parse failed.
118
+ """
119
+ error_msg = (
120
+ f"Error parsing tool arguments as JSON: {str(e)}. "
121
+ "Please ensure the tool call arguments are valid JSON and try again."
122
+ )
123
+ content = TextContent(text=error_msg)
124
+ author = Author(role=Role.TOOL, name=last_msg.recipient)
125
+ return [
126
+ Message(
127
+ author=author,
128
+ content=[content],
129
+ recipient=Role.ASSISTANT,
130
+ channel=last_msg.channel,
131
+ )
132
+ ]
133
+
134
+
135
class SimpleContext(ConversationContext):
136
def __init__(self):
137
self.last_output = None
@@ -339,7 +362,13 @@ class HarmonyContext(ConversationContext):
362
if isinstance(tool_session, Tool):
363
return await tool_session.get_result(self)
364
tool_name = last_msg.recipient.split(".")[1]
342
- args = json.loads(last_msg.content[0].text)
365
+ if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY:
366
+ try:
367
+ args = json.loads(last_msg.content[0].text)
368
+ except json.JSONDecodeError as e:
369
+ return _create_json_parse_error_messages(last_msg, e)
370
+ else:
371
+ args = json.loads(last_msg.content[0].text)
372
result = await tool_session.call_tool(tool_name, args)
373
result_str = result.content[0].text
374
content = TextContent(text=result_str)
@@ -420,7 +449,13 @@ class HarmonyContext(ConversationContext):
449
if isinstance(tool_session, Tool):
450
return await tool_session.get_result(self)
451
tool_name = last_msg.recipient.split(".")[1].split(" ")[0]
423
- args = json.loads(last_msg.content[0].text)
452
+ if envs.VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY:
453
+ try:
454
+ args = json.loads(last_msg.content[0].text)
455
+ except json.JSONDecodeError as e:
456
+ return _create_json_parse_error_messages(last_msg, e)
457
+ else:
458
+ args = json.loads(last_msg.content[0].text)
459
result = await tool_session.call_tool(tool_name, args)
460
result_str = result.content[0].text
461
content = TextContent(text=result_str)
vllm/entrypoints/harmony_utils.py
+18
-1
@@ -340,7 +340,24 @@ def parse_output_message(message: Message) -> list[ResponseOutputItem]:
340
if len(message.content) != 1:
341
raise ValueError("Invalid number of contents in browser message")
342
content = message.content[0]
343
- browser_call = json.loads(content.text)
343
+ # We do not need to check the VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY
344
+ # env variable since if it is not set, we are certain the json is valid
345
+ # The use of Actions for web search will be removed entirely in
346
+ # the future, so this is only necessary temporarily
347
+ try:
348
+ browser_call = json.loads(content.text)
349
+ except json.JSONDecodeError:
350
+ # If the content is not valid JSON, then it was
351
+ # caught and retried by vLLM, which means we
352
+ # need to make note of that so the user is aware
353
+ json_retry_output_message = (
354
+ f"Invalid JSON args, caught and retried: {content.text}"
355
+ )
356
+ browser_call = {
357
+ "query": json_retry_output_message,
358
+ "url": json_retry_output_message,
359
+ "pattern": json_retry_output_message,
360
+ }
361
# TODO: translate to url properly!
362
if recipient == "browser.search":
363
action = ActionSearch(
vllm/envs.py
+7
@@ -199,6 +199,7 @@ if TYPE_CHECKING:
199
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = False
200
VLLM_TUNED_CONFIG_FOLDER: str | None = None
201
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
202
+ VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
203
VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
204
VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False
205
VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
@@ -1331,6 +1332,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
1332
"VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool(
1333
int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0"))
1334
),
1335
+ # Enable automatic retry when tool call JSON parsing fails
1336
+ # If enabled, returns an error message to the model to retry
1337
+ # If disabled (default), raises an exception and fails the request
1338
+ "VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool(
1339
+ int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0"))
1340
+ ),
1341
# Add optional custom scopes for profiling, disable to avoid overheads
1342
"VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool(
1343
int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0"))