A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[Bugfix] Fix tool_choice="none" being ignored by GPT-OSS/harmony models (#30867)
Signed-off-by: yujiepu <pyjapple@gmail.com> Signed-off-by: PlatinumGod <pyjapple@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
PlatinumGod committed
Dec 19, 2025 at 09:34 UTC
6a09612b2e0e09d037a220ea8115632b8084e008
2 files changed
+86
-4
tests/entrypoints/openai/test_serving_chat.py
+77
-1
@@ -52,8 +52,19 @@ def with_tool_parser(request) -> bool:
52
return request.param
53
54
55
+@pytest.fixture(
56
+ scope="module",
57
+ params=[True],
58
+ ids=["exclude_tools_when_tool_choice_none"],
59
+)
60
+def exclude_tools_when_tool_choice_none(request) -> bool:
61
+ return request.param
62
+
63
+
64
@pytest.fixture(scope="module")
56
-def default_server_args(with_tool_parser: bool):
65
+def default_server_args(
66
+ with_tool_parser: bool, exclude_tools_when_tool_choice_none: bool
67
+):
68
args = [
69
# use half precision for speed and memory savings in CI environment
70
"--enforce-eager",
@@ -72,6 +83,8 @@ def default_server_args(with_tool_parser: bool):
83
"--enable-auto-tool-choice",
84
]
85
)
86
+ if exclude_tools_when_tool_choice_none:
87
+ args.append("--exclude-tools-when-tool-choice-none")
88
return args
89
90
@@ -335,6 +348,69 @@ async def test_gpt_oss_tool_message_array_content(
348
assert response_multi_array.choices[0].message is not None
349
350
351
+@pytest.mark.asyncio
352
+async def test_gpt_oss_tool_choice_none(
353
+ gptoss_client: OpenAI,
354
+ with_tool_parser: bool,
355
+ exclude_tools_when_tool_choice_none: bool,
356
+):
357
+ if not (with_tool_parser and exclude_tools_when_tool_choice_none):
358
+ pytest.skip(
359
+ "skip tool_choice tests when non-tool or "
360
+ "--exclude-tools-when-tool-choice-none not set"
361
+ )
362
+
363
+ tools = [
364
+ {
365
+ "type": "function",
366
+ "function": {
367
+ "name": "get_current_weather",
368
+ "description": "Get the current weather in a given location",
369
+ "parameters": {
370
+ "type": "object",
371
+ "properties": {
372
+ "city": {"type": "string"},
373
+ "state": {"type": "string"},
374
+ "unit": {
375
+ "type": "string",
376
+ "enum": ["celsius", "fahrenheit"],
377
+ },
378
+ },
379
+ "required": ["city", "state", "unit"],
380
+ },
381
+ },
382
+ }
383
+ ]
384
+
385
+ messages = [
386
+ {
387
+ "role": "user",
388
+ "content": "What's the temperature(in degrees Celsius) in Dallas?",
389
+ },
390
+ ]
391
+
392
+ tool_choice_auto = await gptoss_client.chat.completions.create(
393
+ model=GPT_OSS_MODEL_NAME,
394
+ messages=messages,
395
+ tools=tools,
396
+ tool_choice="auto",
397
+ temperature=0.0,
398
+ )
399
+ msg = tool_choice_auto.choices[0].message
400
+ assert len(msg.tool_calls) == 1
401
+
402
+ tool_choice_none = await gptoss_client.chat.completions.create(
403
+ model=GPT_OSS_MODEL_NAME,
404
+ messages=messages,
405
+ tools=tools,
406
+ tool_choice="none",
407
+ temperature=0.0,
408
+ )
409
+
410
+ msg = tool_choice_none.choices[0].message
411
+ assert len(msg.tool_calls) == 0
412
+
413
+
414
MODEL_NAME = "openai-community/gpt2"
415
MODEL_NAME_SHORT = "gpt2"
416
CHAT_TEMPLATE = "Dummy chat template for testing {}"
vllm/entrypoints/openai/serving_chat.py
+9
-3
@@ -299,7 +299,10 @@ class OpenAIServingChat(OpenAIServing):
299
)
300
else:
301
# For GPT-OSS.
302
- conversation, engine_prompts = self._make_request_with_harmony(request)
302
+ should_include_tools = tool_dicts is not None
303
+ conversation, engine_prompts = self._make_request_with_harmony(
304
+ request, should_include_tools
305
+ )
306
except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
307
logger.exception("Error in preprocessing prompt inputs")
308
return self.create_error_response(f"{e} {e.__cause__}")
@@ -1833,6 +1836,7 @@ class OpenAIServingChat(OpenAIServing):
1836
def _make_request_with_harmony(
1837
self,
1838
request: ChatCompletionRequest,
1839
+ should_include_tools: bool = True,
1840
):
1841
messages: list[OpenAIMessage] = []
1842
@@ -1850,12 +1854,14 @@ class OpenAIServingChat(OpenAIServing):
1854
reasoning_effort=request.reasoning_effort,
1855
browser_description=None,
1856
python_description=None,
1853
- with_custom_tools=request.tools is not None,
1857
+ with_custom_tools=should_include_tools,
1858
)
1859
messages.append(sys_msg)
1860
1861
# Add developer message.
1858
- dev_msg = get_developer_message(tools=request.tools)
1862
+ dev_msg = get_developer_message(
1863
+ tools=request.tools if should_include_tools else None
1864
+ )
1865
messages.append(dev_msg)
1866
1867
# Add user message.