A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 22:00 UTC
[ROCm][CI] Stabilize Granite tool-use and test URL construction (#43017)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
May 22, 2026 at 23:21 UTC
76ea1d5d2fa3d7daef2b7503e831cdeb8a4fe0ea
5 files changed
+46
-11
examples/tool_chat_template_granite.jinja
+4
@@ -1,4 +1,8 @@
1
{%- if tools %}
2
+ {%- if messages and messages[0]['role'] != 'system' %}
3
+ {{- '<|start_of_role|>system<|end_of_role|>You are a helpful AI assistant with access to the following tools. When a tool is required to answer the user query, respond with <|tool_call|> followed by a JSON list of tools used.<|end_of_text|>
4
+' }}
5
+ {%- endif %}
6
{{- '<|start_of_role|>available_tools<|end_of_role|>
7
' }}
8
{%- for tool in tools %}
tests/tool_use/test_parallel_tool_calls.py
+28
-7
@@ -13,9 +13,19 @@ from .utils import (
13
SEED,
14
WEATHER_TOOL,
15
ServerConfig,
16
+ ensure_system_prompt,
17
)
18
19
20
+def apply_parallel_tool_system_prompt(
21
+ messages,
22
+ server_config: ServerConfig,
23
+):
24
+ if server_config["model"] == "ibm-granite/granite-3.0-8b-instruct":
25
+ return ensure_system_prompt(messages, server_config)
26
+ return messages
27
+
28
+
29
# test: getting the model to generate parallel tool calls (streaming/not)
30
# when requested. NOTE that not all models may support this, so some exclusions
31
# may be added in the future. e.g. llama 3.1 models are not designed to support
@@ -33,8 +43,11 @@ async def test_parallel_tool_calls(
43
44
models = await client.models.list()
45
model_name: str = models.data[0].id
46
+ messages = apply_parallel_tool_system_prompt(
47
+ MESSAGES_ASKING_FOR_PARALLEL_TOOLS, server_config
48
+ )
49
chat_completion = await client.chat.completions.create(
37
- messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
50
+ messages=messages,
51
temperature=0,
52
max_completion_tokens=200,
53
model=model_name,
@@ -73,7 +86,7 @@ async def test_parallel_tool_calls(
86
# make the same request, streaming
87
stream = await client.chat.completions.create(
88
model=model_name,
76
- messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
89
+ messages=messages,
90
temperature=0,
91
max_completion_tokens=200,
92
tools=[WEATHER_TOOL, SEARCH_TOOL],
@@ -162,8 +175,11 @@ async def test_parallel_tool_calls_with_results(
175
176
models = await client.models.list()
177
model_name: str = models.data[0].id
178
+ messages = apply_parallel_tool_system_prompt(
179
+ MESSAGES_WITH_PARALLEL_TOOL_RESPONSE, server_config
180
+ )
181
chat_completion = await client.chat.completions.create(
166
- messages=MESSAGES_WITH_PARALLEL_TOOL_RESPONSE,
182
+ messages=messages,
183
temperature=0,
184
max_completion_tokens=200,
185
model=model_name,
@@ -182,7 +198,7 @@ async def test_parallel_tool_calls_with_results(
198
assert "78" in choice.message.content # Orlando temp in tool response
199
200
stream = await client.chat.completions.create(
185
- messages=MESSAGES_WITH_PARALLEL_TOOL_RESPONSE,
201
+ messages=messages,
202
temperature=0,
203
max_completion_tokens=200,
204
model=model_name,
@@ -220,15 +236,20 @@ async def test_parallel_tool_calls_with_results(
236
237
238
@pytest.mark.asyncio
223
-async def test_parallel_tool_calls_false(client: openai.AsyncOpenAI):
239
+async def test_parallel_tool_calls_false(
240
+ client: openai.AsyncOpenAI, server_config: ServerConfig
241
+):
242
"""
243
Ensure only one tool call is returned when parallel_tool_calls is False.
244
"""
245
246
models = await client.models.list()
247
model_name: str = models.data[0].id
248
+ messages = apply_parallel_tool_system_prompt(
249
+ MESSAGES_ASKING_FOR_PARALLEL_TOOLS, server_config
250
+ )
251
chat_completion = await client.chat.completions.create(
231
- messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
252
+ messages=messages,
253
temperature=0,
254
max_completion_tokens=200,
255
model=model_name,
@@ -248,7 +269,7 @@ async def test_parallel_tool_calls_false(client: openai.AsyncOpenAI):
269
# make the same request, streaming
270
stream = await client.chat.completions.create(
271
model=model_name,
251
- messages=MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
272
+ messages=messages,
273
temperature=0,
274
max_completion_tokens=200,
275
tools=[WEATHER_TOOL, SEARCH_TOOL],
tests/tool_use/test_tool_calls.py
+8
-3
@@ -12,17 +12,22 @@ from .utils import (
12
SEARCH_TOOL,
13
SEED,
14
WEATHER_TOOL,
15
+ ServerConfig,
16
+ ensure_system_prompt,
17
)
18
19
20
# test: request a chat completion that should return tool calls, so we know they
21
# are parsable
22
@pytest.mark.asyncio
21
-async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
23
+async def test_tool_call_and_choice(
24
+ client: openai.AsyncOpenAI, server_config: ServerConfig
25
+):
26
models = await client.models.list()
27
model_name: str = models.data[0].id
28
+ messages = ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config)
29
chat_completion = await client.chat.completions.create(
25
- messages=MESSAGES_ASKING_FOR_TOOLS,
30
+ messages=messages,
31
temperature=0,
32
max_completion_tokens=100,
33
model=model_name,
@@ -68,7 +73,7 @@ async def test_tool_call_and_choice(client: openai.AsyncOpenAI):
73
# make the same request, streaming
74
stream = await client.chat.completions.create(
75
model=model_name,
71
- messages=MESSAGES_ASKING_FOR_TOOLS,
76
+ messages=messages,
77
temperature=0,
78
max_completion_tokens=100,
79
tools=[WEATHER_TOOL, SEARCH_TOOL],
tests/tool_use/utils.py
+4
@@ -201,6 +201,10 @@ CONFIGS: dict[str, ServerConfig] = {
201
"--chat-template",
202
str(VLLM_PATH / "examples/tool_chat_template_granite.jinja"),
203
],
204
+ "system_prompt": "You are a helpful AI assistant with access to tools. "
205
+ "Use two-letter US state abbreviations in weather tool arguments. "
206
+ "When a tool is required to answer the user query, respond with "
207
+ "<|tool_call|> followed by a JSON list of tools used.",
208
},
209
"granite-3.1-8b": {
210
"model": "ibm-granite/granite-3.1-8b-instruct",
tests/utils.py
+2
-1
@@ -660,7 +660,8 @@ class RemoteVLLMServer:
660
)
661
662
def url_for(self, *parts: str) -> str:
663
- return self.url_root + "/" + "/".join(parts)
663
+ path = "/".join(part.strip("/") for part in parts if part)
664
+ return f"{self.url_root}/{path}"
665
666
def get_client(self, **kwargs):
667
if "timeout" not in kwargs: