A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[BugFix] Enforce Mistral ToolCall id constraint when using the Mistral tool call parser (#9020)
Guillaume Calmettes committed
Oct 3, 2024 at 10:44 UTC
83caf35e082b2657dce5f71ff965a13653a763b0
3 files changed
+22
-6
tests/tool_use/test_parallel_tool_calls.py
+2
-2
@@ -45,7 +45,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
45
assert tool_call.type == "function"
46
assert tool_call.function is not None
47
assert isinstance(tool_call.id, str)
48
- assert len(tool_call.id) > 16
48
+ assert len(tool_call.id) >= 9
49
50
# make sure the weather tool was called correctly
51
assert tool_call.function.name == WEATHER_TOOL["function"]["name"]
@@ -108,7 +108,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
108
if tool_call.id:
109
tool_call_id_count += 1
110
assert (isinstance(tool_call.id, str)
111
- and (len(tool_call.id) > 16))
111
+ and (len(tool_call.id) >= 9))
112
113
# if parts of the function start being streamed
114
if tool_call.function:
tests/tool_use/test_tool_calls.py
+2
-2
@@ -33,7 +33,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
33
assert tool_calls[0].type == 'function'
34
assert tool_calls[0].function is not None
35
assert isinstance(tool_calls[0].id, str)
36
- assert len(tool_calls[0].id) > 16
36
+ assert len(tool_calls[0].id) >= 9
37
38
# make sure the weather tool was called (classic example) with arguments
39
assert tool_calls[0].function.name == WEATHER_TOOL["function"]["name"]
@@ -106,7 +106,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
106
107
assert finish_reason_count == 1
108
assert role_name == 'assistant'
109
- assert isinstance(tool_call_id, str) and (len(tool_call_id) > 16)
109
+ assert isinstance(tool_call_id, str) and (len(tool_call_id) >= 9)
110
111
# validate the name and arguments
112
assert function_name == WEATHER_TOOL["function"]["name"]
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+18
-2
@@ -1,9 +1,12 @@
1
import json
2
import re
3
+from random import choices
4
+from string import ascii_letters, digits
5
from typing import Dict, List, Sequence, Union
6
7
import partial_json_parser
8
from partial_json_parser.core.options import Allow
9
+from pydantic import Field
10
11
from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
12
DeltaToolCall,
@@ -19,6 +22,19 @@ from vllm.utils import random_uuid
22
23
logger = init_logger(__name__)
24
25
+ALPHANUMERIC = ascii_letters + digits
26
+
27
+
28
+class MistralToolCall(ToolCall):
29
+ id: str = Field(
30
+ default_factory=lambda: MistralToolCall.generate_random_id())
31
+
32
+ @staticmethod
33
+ def generate_random_id():
34
+ # Mistral Tool Call Ids must be alphanumeric with a maximum length of 9.
35
+ # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299
36
+ return "".join(choices(ALPHANUMERIC, k=9))
37
+
38
39
class MistralToolParser(ToolParser):
40
"""
@@ -71,8 +87,8 @@ class MistralToolParser(ToolParser):
87
# load the JSON, and then use it to build the Function and
88
# Tool Call
89
function_call_arr = json.loads(raw_tool_call)
74
- tool_calls: List[ToolCall] = [
75
- ToolCall(
90
+ tool_calls: List[MistralToolCall] = [
91
+ MistralToolCall(
92
type="function",
93
function=FunctionCall(
94
name=raw_function_call["name"],