A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[BugFix] Do not raise a `ValueError` when `tool_choice` is set to the supported `none` option and `tools` are not defined. (#10000)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Nov 12, 2024 at 12:13 UTC
36c513a0762b104c9076ab6a3449ea3efff6db4d
3 files changed
+19
-6
docs/source/serving/openai_compatible_server.md
+2
-2
@@ -215,10 +215,10 @@ The order of priorities is `command line > config file values > defaults`.
215
---
216
217
## Tool calling in the chat completion API
218
-
219
-vLLM supports named function calling and `auto` tool choice in the chat completion API. The `tool_choice` options `required` is **not yet supported** but on the roadmap.
218
+vLLM currently supports named function calling, as well as the `auto` and `none` options for the `tool_choice` field in the chat completion API. The `tool_choice` option `required` is **not yet supported** but on the roadmap.
219
220
It is the callers responsibility to prompt the model with the tool information, vLLM will not automatically manipulate the prompt.
221
+Please see below for recommended configuration and chat templates to use when function calling is to be used with the different models.
222
223
224
### Named Function Calling
vllm/entrypoints/openai/protocol.py
+8
-2
@@ -454,6 +454,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
454
if "tool_choice" not in data and data.get("tools"):
455
data["tool_choice"] = "auto"
456
457
+ # if "tool_choice" is "none" -- ignore tools if present
458
+ if "tool_choice" in data and data["tool_choice"] == "none":
459
+ # ensure that no tools are present
460
+ data.pop("tools", None)
461
+ return data
462
+
463
# if "tool_choice" is specified -- validation
464
if "tool_choice" in data:
465
@@ -467,8 +473,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
473
if data["tool_choice"] != "auto" and not isinstance(
474
data["tool_choice"], dict):
475
raise ValueError(
470
- "`tool_choice` must either be a named tool or \"auto\". "
471
- "`tool_choice=\"none\" is not supported.")
476
+ "`tool_choice` must either be a named tool, \"auto\", "
477
+ "or \"none\".")
478
479
# ensure that if "tool_choice" is specified as an object,
480
# it matches a valid tool
vllm/entrypoints/openai/serving_engine.py
+9
-2
@@ -469,12 +469,19 @@ class OpenAIServing:
469
470
mm_data = await mm_data_future
471
472
- if tool_parser is not None:
472
+ # tool parsing is done only if a tool_parser has been set and if
473
+ # tool_choice is not "none" (if tool_choice is "none" but a tool_parser
474
+ # is set, we want to prevent parsing a tool_call hallucinated by the LLM
475
+ should_parse_tools = tool_parser is not None and (hasattr(
476
+ request, "tool_choice") and request.tool_choice != "none")
477
+
478
+ if should_parse_tools:
479
if not isinstance(request, ChatCompletionRequest):
480
msg = "Tool usage is only supported for Chat Completions API"
481
raise NotImplementedError(msg)
482
477
- request = tool_parser(tokenizer).adjust_request(request=request)
483
+ request = tool_parser(tokenizer).adjust_request( # type: ignore
484
+ request=request)
485
486
if isinstance(request_prompt, str):
487
prompt_inputs = self._tokenize_prompt_input(