@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 83caf35e08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[BugFix] Enforce Mistral ToolCall id constraint when using the Mistral tool call parser (#9020)

Guillaume Calmettes committed Oct 3, 2024 at 10:44 UTC 83caf35e082b2657dce5f71ff965a13653a763b0
3 files changed +22 -6
tests/tool_use/test_parallel_tool_calls.py
+2 -2
@@ -45,7 +45,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
45 assert tool_call.type == "function"
46 assert tool_call.function is not None
47 assert isinstance(tool_call.id, str)
48 - assert len(tool_call.id) > 16
48 + assert len(tool_call.id) >= 9
49
50 # make sure the weather tool was called correctly
51 assert tool_call.function.name == WEATHER_TOOL["function"]["name"]
@@ -108,7 +108,7 @@ async def test_parallel_tool_calls(client: openai.AsyncOpenAI,
108 if tool_call.id:
109 tool_call_id_count += 1
110 assert (isinstance(tool_call.id, str)
111 - and (len(tool_call.id) > 16))
111 + and (len(tool_call.id) >= 9))
112
113 # if parts of the function start being streamed
114 if tool_call.function:
tests/tool_use/test_tool_calls.py
+2 -2
@@ -33,7 +33,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
33 assert tool_calls[0].type == 'function'
34 assert tool_calls[0].function is not None
35 assert isinstance(tool_calls[0].id, str)
36 - assert len(tool_calls[0].id) > 16
36 + assert len(tool_calls[0].id) >= 9
37
38 # make sure the weather tool was called (classic example) with arguments
39 assert tool_calls[0].function.name == WEATHER_TOOL["function"]["name"]
@@ -106,7 +106,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
106
107 assert finish_reason_count == 1
108 assert role_name == 'assistant'
109 - assert isinstance(tool_call_id, str) and (len(tool_call_id) > 16)
109 + assert isinstance(tool_call_id, str) and (len(tool_call_id) >= 9)
110
111 # validate the name and arguments
112 assert function_name == WEATHER_TOOL["function"]["name"]
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+18 -2
@@ -1,9 +1,12 @@
1 import json
2 import re
3 +from random import choices
4 +from string import ascii_letters, digits
5 from typing import Dict, List, Sequence, Union
6
7 import partial_json_parser
8 from partial_json_parser.core.options import Allow
9 +from pydantic import Field
10
11 from vllm.entrypoints.openai.protocol import (DeltaFunctionCall, DeltaMessage,
12 DeltaToolCall,
@@ -19,6 +22,19 @@ from vllm.utils import random_uuid
22
23 logger = init_logger(__name__)
24
25 +ALPHANUMERIC = ascii_letters + digits
26 +
27 +
28 +class MistralToolCall(ToolCall):
29 + id: str = Field(
30 + default_factory=lambda: MistralToolCall.generate_random_id())
31 +
32 + @staticmethod
33 + def generate_random_id():
34 + # Mistral Tool Call Ids must be alphanumeric with a maximum length of 9.
35 + # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299
36 + return "".join(choices(ALPHANUMERIC, k=9))
37 +
38
39 class MistralToolParser(ToolParser):
40 """
@@ -71,8 +87,8 @@ class MistralToolParser(ToolParser):
87 # load the JSON, and then use it to build the Function and
88 # Tool Call
89 function_call_arr = json.loads(raw_tool_call)
74 - tool_calls: List[ToolCall] = [
75 - ToolCall(
90 + tool_calls: List[MistralToolCall] = [
91 + MistralToolCall(
92 type="function",
93 function=FunctionCall(
94 name=raw_function_call["name"],