| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from copy import deepcopy |
| 5 | from typing import Any |
| 6 | |
| 7 | from openai.types.chat import ChatCompletionMessageParam, ChatCompletionToolParam |
| 8 | from typing_extensions import TypedDict |
| 9 | |
| 10 | from tests.utils import VLLM_PATH |
| 11 | |
| 12 | |
| 13 | class ServerConfig(TypedDict, total=False): |
| 14 | model: str |
| 15 | arguments: list[str] |
| 16 | system_prompt: str | None |
| 17 | supports_parallel: bool | None |
| 18 | supports_rocm: bool | None |
| 19 | extended: bool | None # tests do not run in CI automatically |
| 20 | |
| 21 | |
| 22 | def patch_system_prompt( |
| 23 | messages: list[dict[str, Any]], system_prompt: str |
| 24 | ) -> list[dict[str, Any]]: |
| 25 | new_messages = deepcopy(messages) |
| 26 | if new_messages[0]["role"] == "system": |
| 27 | new_messages[0]["content"] = system_prompt |
| 28 | else: |
| 29 | new_messages.insert(0, {"role": "system", "content": system_prompt}) |
| 30 | return new_messages |
| 31 | |
| 32 | |
| 33 | def ensure_system_prompt( |
| 34 | messages: list[dict[str, Any]], config: ServerConfig |
| 35 | ) -> list[dict[str, Any]]: |
| 36 | prompt = config.get("system_prompt") |
| 37 | if prompt: |
| 38 | return patch_system_prompt(messages, prompt) |
| 39 | else: |
| 40 | return messages |
| 41 | |
| 42 | |
| 43 | # universal args for all models go here. also good if you need to test locally |
| 44 | # and change type or KV cache quantization or something. |
| 45 | SEED = 42 |
| 46 | |
| 47 | ARGS: list[str] = [ |
| 48 | "--enable-auto-tool-choice", |
| 49 | "--max-model-len", |
| 50 | "1024", |
| 51 | "--max-num-seqs", |
| 52 | "256", |
| 53 | ] |
| 54 | |
| 55 | CONFIGS: dict[str, ServerConfig] = { |
| 56 | "hermes": { |
| 57 | "model": "NousResearch/Hermes-3-Llama-3.1-8B", |
| 58 | "arguments": [ |
| 59 | "--enforce-eager", |
| 60 | "--no-enable-prefix-caching", |
| 61 | "--tool-call-parser", |
| 62 | "hermes", |
| 63 | "--chat-template", |
| 64 | str(VLLM_PATH / "examples/tool_chat_template_hermes.jinja"), |
| 65 | ], |
| 66 | "system_prompt": "You are a helpful assistant with access to tools. If a tool" |
| 67 | " that you have would be helpful to answer a user query, " |
| 68 | "call the tool. Otherwise, answer the user's query directly " |
| 69 | "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT " |
| 70 | "to the user's question - just respond to it normally.", |
| 71 | }, |
| 72 | "llama": { |
| 73 | "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", |
| 74 | "arguments": [ |
| 75 | "--enforce-eager", |
| 76 | "--no-enable-prefix-caching", |
| 77 | "--tool-call-parser", |
| 78 | "llama3_json", |
| 79 | "--chat-template", |
| 80 | str(VLLM_PATH / "examples/tool_chat_template_llama3.1_json.jinja"), |
| 81 | ], |
| 82 | "supports_parallel": False, |
| 83 | }, |
| 84 | "llama3.2": { |
| 85 | "model": "meta-llama/Llama-3.2-3B-Instruct", |
| 86 | "arguments": [ |
| 87 | "--enforce-eager", |
| 88 | "--no-enable-prefix-caching", |
| 89 | "--tool-call-parser", |
| 90 | "llama3_json", |
| 91 | "--chat-template", |
| 92 | str(VLLM_PATH / "examples/tool_chat_template_llama3.2_json.jinja"), |
| 93 | ], |
| 94 | "supports_parallel": False, |
| 95 | }, |
| 96 | "llama4": { |
| 97 | "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", |
| 98 | "arguments": [ |
| 99 | "--enforce-eager", |
| 100 | "--no-enable-prefix-caching", |
| 101 | "--tool-call-parser", |
| 102 | "llama4_pythonic", |
| 103 | "--chat-template", |
| 104 | str(VLLM_PATH / "examples/tool_chat_template_llama4_pythonic.jinja"), |
| 105 | "-tp", |
| 106 | "4", |
| 107 | ], |
| 108 | "supports_parallel": False, |
| 109 | "extended": True, |
| 110 | }, |
| 111 | "llama4_json": { |
| 112 | "model": "meta-llama/Llama-4-Scout-17B-16E-Instruct", |
| 113 | "arguments": [ |
| 114 | "--enforce-eager", |
| 115 | "--no-enable-prefix-caching", |
| 116 | "-tp", |
| 117 | "4", |
| 118 | "--distributed-executor-backend", |
| 119 | "mp", |
| 120 | "--tool-call-parser", |
| 121 | "llama4_json", |
| 122 | "--chat-template", |
| 123 | str(VLLM_PATH / "examples/tool_chat_template_llama4_json.jinja"), |
| 124 | ], |
| 125 | "supports_parallel": True, |
| 126 | "extended": True, |
| 127 | }, |
| 128 | "mistral-7b": { |
| 129 | "model": "mistralai/Mistral-7B-Instruct-v0.3", |
| 130 | "arguments": [ |
| 131 | "--enforce-eager", |
| 132 | "--no-enable-prefix-caching", |
| 133 | "--tokenizer_mode", |
| 134 | "hf", |
| 135 | "--load_format", |
| 136 | "hf", |
| 137 | "--config_format", |
| 138 | "hf", |
| 139 | "--tool-call-parser", |
| 140 | "mistral", |
| 141 | "--chat-template", |
| 142 | str(VLLM_PATH / "examples/tool_chat_template_mistral.jinja"), |
| 143 | '--ignore-patterns="consolidated.safetensors"', |
| 144 | ], |
| 145 | "system_prompt": "You are a helpful assistant with access to tools. If a tool" |
| 146 | " that you have would be helpful to answer a user query, " |
| 147 | "call the tool. Otherwise, answer the user's query directly " |
| 148 | "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT " |
| 149 | "to the user's question - just respond to it normally.", |
| 150 | "supports_parallel": True, |
| 151 | }, |
| 152 | "mistral-small-3.2": { |
| 153 | "model": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", |
| 154 | "arguments": [ |
| 155 | "--enforce-eager", |
| 156 | "--no-enable-prefix-caching", |
| 157 | "--tool-call-parser", |
| 158 | "mistral", |
| 159 | "--tokenizer-mode", |
| 160 | "mistral", |
| 161 | "--config-format", |
| 162 | "mistral", |
| 163 | "--load-format", |
| 164 | "mistral", |
| 165 | "--tensor-parallel-size", |
| 166 | "4", |
| 167 | '--ignore-patterns="consolidated.safetensors"', |
| 168 | ], |
| 169 | "system_prompt": "You are a helpful assistant with access to tools. If a tool" |
| 170 | " that you have would be helpful to answer a user query, " |
| 171 | "call the tool. Otherwise, answer the user's query directly " |
| 172 | "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT " |
| 173 | "to the user's question - just respond to it normally.", |
| 174 | "supports_parallel": True, |
| 175 | "extended": True, |
| 176 | }, |
| 177 | # FIXME: This test currently fails, need to debug why. |
| 178 | # "granite20b": { |
| 179 | # "model": "mbayser/granite-20b-functioncalling-FP8-KV", |
| 180 | # "arguments": [ |
| 181 | # "--tool-call-parser", |
| 182 | # "granite-20b-fc", |
| 183 | # "--chat-template", |
| 184 | # str(VLLM_PATH / "examples/tool_chat_template_granite_20b_fc.jinja"), |
| 185 | # "--max_num_seqs", |
| 186 | # "1", |
| 187 | # "--enforce-eager", |
| 188 | # "--cpu-offload-gb", |
| 189 | # "20", |
| 190 | # ], |
| 191 | # "supports_parallel": False, |
| 192 | # "supports_rocm": False, |
| 193 | # }, |
| 194 | "granite-3.0-8b": { |
| 195 | "model": "ibm-granite/granite-3.0-8b-instruct", |
| 196 | "arguments": [ |
| 197 | "--enforce-eager", |
| 198 | "--no-enable-prefix-caching", |
| 199 | "--tool-call-parser", |
| 200 | "granite", |
| 201 | "--chat-template", |
| 202 | str(VLLM_PATH / "examples/tool_chat_template_granite.jinja"), |
| 203 | ], |
| 204 | "system_prompt": "You are a helpful AI assistant with access to tools. " |
| 205 | "Use two-letter US state abbreviations in weather tool arguments. " |
| 206 | "When a tool is required to answer the user query, respond with " |
| 207 | "<|tool_call|> followed by a JSON list of tools used.", |
| 208 | }, |
| 209 | "granite-3.1-8b": { |
| 210 | "model": "ibm-granite/granite-3.1-8b-instruct", |
| 211 | "arguments": [ |
| 212 | "--enforce-eager", |
| 213 | "--no-enable-prefix-caching", |
| 214 | "--tool-call-parser", |
| 215 | "granite", |
| 216 | ], |
| 217 | "supports_parallel": True, |
| 218 | }, |
| 219 | "internlm": { |
| 220 | "model": "internlm/internlm2_5-7b-chat", |
| 221 | "arguments": [ |
| 222 | "--enforce-eager", |
| 223 | "--no-enable-prefix-caching", |
| 224 | "--tool-call-parser", |
| 225 | "internlm", |
| 226 | "--chat-template", |
| 227 | str(VLLM_PATH / "examples/tool_chat_template_internlm2_tool.jinja"), |
| 228 | "--trust_remote_code", |
| 229 | ], |
| 230 | "supports_parallel": False, |
| 231 | }, |
| 232 | "toolACE": { |
| 233 | "model": "Team-ACE/ToolACE-8B", |
| 234 | "arguments": [ |
| 235 | "--enforce-eager", |
| 236 | "--no-enable-prefix-caching", |
| 237 | "--tool-call-parser", |
| 238 | "pythonic", |
| 239 | "--chat-template", |
| 240 | str(VLLM_PATH / "examples/tool_chat_template_toolace.jinja"), |
| 241 | ], |
| 242 | "supports_parallel": True, |
| 243 | }, |
| 244 | } |
| 245 | |
| 246 | WEATHER_TOOL: ChatCompletionToolParam = { |
| 247 | "type": "function", |
| 248 | "function": { |
| 249 | "name": "get_current_weather", |
| 250 | "description": "Get the current weather in a given location", |
| 251 | "parameters": { |
| 252 | "type": "object", |
| 253 | "properties": { |
| 254 | "city": { |
| 255 | "type": "string", |
| 256 | "description": "The city to find the weather for, " |
| 257 | "e.g. 'San Francisco'", |
| 258 | }, |
| 259 | "state": { |
| 260 | "type": "string", |
| 261 | "description": "must the two-letter abbreviation for the state " |
| 262 | "that the city is in, e.g. 'CA' which would " |
| 263 | "mean 'California'", |
| 264 | }, |
| 265 | "unit": { |
| 266 | "type": "string", |
| 267 | "description": "The unit to fetch the temperature in", |
| 268 | "enum": ["celsius", "fahrenheit"], |
| 269 | }, |
| 270 | }, |
| 271 | }, |
| 272 | }, |
| 273 | } |
| 274 | |
| 275 | SEARCH_TOOL: ChatCompletionToolParam = { |
| 276 | "type": "function", |
| 277 | "function": { |
| 278 | "name": "web_search", |
| 279 | "description": "Search the internet and get a summary of the top " |
| 280 | "10 webpages. Should only be used if you don't know " |
| 281 | "the answer to a user query, and the results are likely" |
| 282 | "to be able to be found with a web search", |
| 283 | "parameters": { |
| 284 | "type": "object", |
| 285 | "properties": { |
| 286 | "search_term": { |
| 287 | "type": "string", |
| 288 | "description": "The term to use in the search. This should" |
| 289 | "ideally be keywords to search for, not a" |
| 290 | "natural-language question", |
| 291 | } |
| 292 | }, |
| 293 | "required": ["search_term"], |
| 294 | }, |
| 295 | }, |
| 296 | } |
| 297 | |
| 298 | MESSAGES_WITHOUT_TOOLS: list[ChatCompletionMessageParam] = [ |
| 299 | {"role": "user", "content": "Hi! How are you?"}, |
| 300 | {"role": "assistant", "content": "I'm doing great! How can I assist you?"}, |
| 301 | {"role": "user", "content": "Can you tell me a joke please?"}, |
| 302 | ] |
| 303 | |
| 304 | MESSAGES_ASKING_FOR_TOOLS: list[ChatCompletionMessageParam] = [ |
| 305 | {"role": "user", "content": "What is the weather in Dallas, Texas in Fahrenheit?"} |
| 306 | ] |
| 307 | |
| 308 | MESSAGES_WITH_TOOL_RESPONSE: list[ChatCompletionMessageParam] = [ |
| 309 | {"role": "user", "content": "What is the weather in Dallas, Texas in Fahrenheit?"}, |
| 310 | { |
| 311 | "role": "assistant", |
| 312 | "tool_calls": [ |
| 313 | { |
| 314 | "id": "chatcmpl-tool-03e6481b146e408e9523d9c956696295", |
| 315 | "type": "function", |
| 316 | "function": { |
| 317 | "name": WEATHER_TOOL["function"]["name"], |
| 318 | "arguments": '{"city": "Dallas", "state": "TX", ' |
| 319 | '"unit": "fahrenheit"}', |
| 320 | }, |
| 321 | } |
| 322 | ], |
| 323 | }, |
| 324 | { |
| 325 | "role": "tool", |
| 326 | "tool_call_id": "chatcmpl-tool-03e6481b146e408e9523d9c956696295", |
| 327 | "content": "The weather in Dallas is 98 degrees fahrenheit, with partly" |
| 328 | "cloudy skies and a low chance of rain.", |
| 329 | }, |
| 330 | ] |
| 331 | |
| 332 | MESSAGES_ASKING_FOR_PARALLEL_TOOLS: list[ChatCompletionMessageParam] = [ |
| 333 | { |
| 334 | "role": "user", |
| 335 | "content": "What is the weather in Dallas, Texas and Orlando, Florida in " |
| 336 | "Fahrenheit?", |
| 337 | } |
| 338 | ] |
| 339 | |
| 340 | MESSAGES_WITH_PARALLEL_TOOL_RESPONSE: list[ChatCompletionMessageParam] = [ |
| 341 | { |
| 342 | "role": "user", |
| 343 | "content": "What is the weather in Dallas, Texas and Orlando, Florida in " |
| 344 | "Fahrenheit?", |
| 345 | }, |
| 346 | { |
| 347 | "role": "assistant", |
| 348 | "tool_calls": [ |
| 349 | { |
| 350 | "id": "chatcmpl-tool-03e6481b146e408e9523d9c956696295", |
| 351 | "type": "function", |
| 352 | "function": { |
| 353 | "name": WEATHER_TOOL["function"]["name"], |
| 354 | "arguments": '{"city": "Dallas", "state": "TX", ' |
| 355 | '"unit": "fahrenheit"}', |
| 356 | }, |
| 357 | }, |
| 358 | { |
| 359 | "id": "chatcmpl-tool-d027061e1bd21cda48bee7da829c1f5b", |
| 360 | "type": "function", |
| 361 | "function": { |
| 362 | "name": WEATHER_TOOL["function"]["name"], |
| 363 | "arguments": '{"city": "Orlando", "state": "Fl", ' |
| 364 | '"unit": "fahrenheit"}', |
| 365 | }, |
| 366 | }, |
| 367 | ], |
| 368 | }, |
| 369 | { |
| 370 | "role": "tool", |
| 371 | "tool_call_id": "chatcmpl-tool-03e6481b146e408e9523d9c956696295", |
| 372 | "content": "The weather in Dallas TX is 98 degrees fahrenheit with mostly " |
| 373 | "cloudy skies and a chance of rain in the evening.", |
| 374 | }, |
| 375 | { |
| 376 | "role": "tool", |
| 377 | "tool_call_id": "chatcmpl-tool-d027061e1bd21cda48bee7da829c1f5b", |
| 378 | "content": "The weather in Orlando FL is 78 degrees fahrenheit with clear" |
| 379 | "skies.", |
| 380 | }, |
| 381 | ] |