@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 314cfade02
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Frontend] Generate valid tool call IDs when using `tokenizer-mode=mistral` (#12332)

Rafael Vasquez committed Feb 12, 2025 at 11:29 UTC 314cfade02b28d50349c4df1a7ea0bbdaef589f1
8 files changed +149 -8
tests/mistral_tool_use/__init__.py
tests/mistral_tool_use/conftest.py new
+40
@@ -0,0 +1,40 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import pytest
4 +import pytest_asyncio
5 +from huggingface_hub import snapshot_download
6 +
7 +from tests.utils import RemoteOpenAIServer
8 +from vllm.platforms import current_platform
9 +
10 +from .utils import ARGS, CONFIGS, ServerConfig
11 +
12 +
13 +# for each server config, download the model and return the config
14 +@pytest.fixture(scope="session", params=CONFIGS.keys())
15 +def server_config(request):
16 + config = CONFIGS[request.param]
17 +
18 + if current_platform.is_rocm() and not config.get("supports_rocm", True):
19 + pytest.skip("The {} model can't be tested on the ROCm platform".format(
20 + config["model"]))
21 +
22 + # download model and tokenizer using transformers
23 + snapshot_download(config["model"])
24 + yield CONFIGS[request.param]
25 +
26 +
27 +# run this for each server config
28 +@pytest.fixture(scope="session")
29 +def server(request, server_config: ServerConfig):
30 + model = server_config["model"]
31 + args_for_model = server_config["arguments"]
32 + with RemoteOpenAIServer(model, ARGS + args_for_model,
33 + max_wait_seconds=480) as server:
34 + yield server
35 +
36 +
37 +@pytest_asyncio.fixture
38 +async def client(server: RemoteOpenAIServer):
39 + async with server.get_async_client() as async_client:
40 + yield async_client
tests/mistral_tool_use/test_mistral_tool_calls.py new
+29
@@ -0,0 +1,29 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import openai
4 +import pytest
5 +
6 +from tests.tool_use.utils import MESSAGES_ASKING_FOR_TOOLS, WEATHER_TOOL
7 +
8 +
9 +# test: a tool_choice with mistral-tokenizer results in an ID of length 9
10 +@pytest.mark.asyncio
11 +async def test_tool_call_with_tool_choice(client: openai.AsyncOpenAI):
12 + models = await client.models.list()
13 + model_name: str = models.data[0].id
14 + chat_completion = await client.chat.completions.create(
15 + messages=MESSAGES_ASKING_FOR_TOOLS,
16 + temperature=0,
17 + max_completion_tokens=100,
18 + model=model_name,
19 + tools=[WEATHER_TOOL],
20 + tool_choice=WEATHER_TOOL,
21 + logprobs=False)
22 +
23 + choice = chat_completion.choices[0]
24 +
25 + assert choice.finish_reason != "tool_calls" # "stop" or "length"
26 + assert choice.message.role == "assistant"
27 + assert choice.message.tool_calls is None \
28 + or len(choice.message.tool_calls) == 1
29 + assert len(choice.message.tool_calls[0].id) == 9 # length of 9 for mistral
tests/mistral_tool_use/utils.py new
+33
@@ -0,0 +1,33 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from typing import Dict, List, Optional
4 +
5 +from typing_extensions import TypedDict
6 +
7 +
8 +class ServerConfig(TypedDict, total=False):
9 + model: str
10 + arguments: List[str]
11 + system_prompt: Optional[str]
12 + supports_parallel: Optional[bool]
13 + supports_rocm: Optional[bool]
14 +
15 +
16 +ARGS: List[str] = ["--max-model-len", "1024"]
17 +
18 +CONFIGS: Dict[str, ServerConfig] = {
19 + "mistral": {
20 + "model":
21 + "mistralai/Mistral-7B-Instruct-v0.3",
22 + "arguments": [
23 + "--tokenizer-mode", "mistral",
24 + "--ignore-patterns=\"consolidated.safetensors\""
25 + ],
26 + "system_prompt":
27 + "You are a helpful assistant with access to tools. If a tool"
28 + " that you have would be helpful to answer a user query, "
29 + "call the tool. Otherwise, answer the user's query directly "
30 + "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
31 + "to the user's question - just respond to it normally."
32 + },
33 +}
vllm/entrypoints/openai/serving_chat.py
+11 -5
@@ -28,12 +28,15 @@ from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser,
28 from vllm.entrypoints.openai.serving_engine import OpenAIServing
29 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
30 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
31 +from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
32 + MistralToolCall)
33 from vllm.logger import init_logger
34 from vllm.outputs import CompletionOutput, RequestOutput
35 from vllm.sampling_params import BeamSearchParams, SamplingParams
36 from vllm.sequence import Logprob
37 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
36 -from vllm.transformers_utils.tokenizers import maybe_serialize_tool_calls
38 +from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
39 + truncate_tool_call_ids)
40
41 logger = init_logger(__name__)
42
@@ -150,11 +153,12 @@ class OpenAIServingChat(OpenAIServing):
153 return self.create_error_response(
154 "tool_choice = \"required\" is not supported!")
155
153 - # because of issues with pydantic we need to potentially
154 - # re-serialize the tool_calls field of the request
155 - # for more info: see comment in `maybe_serialize_tool_calls`
156 if isinstance(tokenizer, MistralTokenizer):
157 + # because of issues with pydantic we need to potentially
158 + # re-serialize the tool_calls field of the request
159 + # for more info: see comment in `maybe_serialize_tool_calls`
160 maybe_serialize_tool_calls(request)
161 + truncate_tool_call_ids(request)
162
163 if (request.tool_choice == "auto" and
164 not (self.enable_auto_tools and tool_parser is not None)
@@ -745,11 +749,13 @@ class OpenAIServingChat(OpenAIServing):
749 elif request.tool_choice and type(
750 request.tool_choice) is ChatCompletionNamedToolChoiceParam:
751
752 + tool_call_class = MistralToolCall if isinstance(
753 + tokenizer, MistralTokenizer) else ToolCall
754 message = ChatMessage(
755 role=role,
756 content="",
757 tool_calls=[
752 - ToolCall(function=FunctionCall(
758 + tool_call_class(function=FunctionCall(
759 name=request.tool_choice.function.name,
760 arguments=output.text))
761 ])
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+1 -1
@@ -33,7 +33,7 @@ class MistralToolCall(ToolCall):
33
34 @staticmethod
35 def generate_random_id():
36 - # Mistral Tool Call Ids must be alphanumeric with a maximum length of 9.
36 + # Mistral Tool Call Ids must be alphanumeric with a length of 9.
37 # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299
38 return "".join(choices(ALPHANUMERIC, k=9))
39
vllm/transformers_utils/tokenizers/__init__.py
+5 -2
@@ -1,5 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from .mistral import MistralTokenizer, maybe_serialize_tool_calls
3 +from .mistral import (MistralTokenizer, maybe_serialize_tool_calls,
4 + truncate_tool_call_ids)
5
5 -__all__ = ["MistralTokenizer", "maybe_serialize_tool_calls"]
6 +__all__ = [
7 + "MistralTokenizer", "maybe_serialize_tool_calls", "truncate_tool_call_ids"
8 +]
vllm/transformers_utils/tokenizers/mistral.py
+30
@@ -68,6 +68,36 @@ def maybe_serialize_tool_calls(request: "ChatCompletionRequest"):
68 request.messages[i]["tool_calls"] = validated_tool_calls
69
70
71 +def truncate_tool_call_ids(request: "ChatCompletionRequest"):
72 + """Truncates tool call IDs for Mistral's ID requirements."""
73 + for i, message in enumerate(request.messages):
74 + if message.get("role") == 'assistant':
75 + tool_calls = message.get("tool_calls", [])
76 + for tool_call in tool_calls:
77 + if len(tool_call["id"]) > 9:
78 + logger.warning(
79 + "Truncating tool call ID: %s to %s",
80 + tool_call["id"],
81 + tool_call["id"][-9:],
82 + )
83 + tool_call["id"] = tool_call["id"][-9:]
84 +
85 + request.messages[i]["tool_calls"] = tool_calls
86 +
87 + elif message.get("role") in {"tool_results", "tool"}:
88 + if "tool_call_id" in message:
89 + tool_call_id = message["tool_call_id"]
90 +
91 + if len(tool_call_id) > 9:
92 + logger.warning(
93 + "Truncating tool_call_id: %s to %s",
94 + tool_call_id,
95 + tool_call_id[-9:],
96 + )
97 + tool_call_id = tool_call_id[-9:]
98 + request.messages[i]["tool_call_id"] = tool_call_id
99 +
100 +
101 def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
102 repo_cache = os.path.join(
103 huggingface_hub.constants.HF_HUB_CACHE,