A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Frontend] Added chat templates for LLaMa4 pythonic tool calling (#16463)
Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com> Co-authored-by: Kai Wu <kaiwu@meta.com>
Ye (Charlotte) Qi committed
Apr 11, 2025 at 15:26 UTC
16eda8c43a9d62f3dc90e8a0149855b3820049c8
5 files changed
+182
-2
docs/source/features/tool_calling.md
+2
@@ -245,6 +245,8 @@ Example supported models:
245
* `meta-llama/Llama-3.2-3B-Instruct`\* (use with `examples/tool_chat_template_llama3.2_pythonic.jinja`)
246
* `Team-ACE/ToolACE-8B` (use with `examples/tool_chat_template_toolace.jinja`)
247
* `fixie-ai/ultravox-v0_4-ToolACE-8B` (use with `examples/tool_chat_template_toolace.jinja`)
248
+* `meta-llama/Llama-4-Scout-17B-16E-Instruct`\* (use with `examples/tool_chat_template_llama4_pythonic.jinja`)
249
+* `meta-llama/Llama-4-Maverick-17B-128E-Instruct`\* (use with `examples/tool_chat_template_llama4_pythonic.jinja`)
250
251
Flags: `--tool-call-parser pythonic --chat-template {see_above}`
252
examples/tool_chat_template_llama4_pythonic.jinja
new
+139
@@ -0,0 +1,139 @@
1
+{{- bos_token }}
2
+{%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+{%- endif %}
5
+{%- if not tools_in_user_message is defined %}
6
+ {%- set tools_in_user_message = false %}
7
+{%- endif %}
8
+{%- if not tools is defined %}
9
+ {%- set tools = none %}
10
+{%- endif %}
11
+
12
+{#- This block extracts the system message, so we can slot it into the right place. #}
13
+{%- if messages[0]['role'] == 'system' %}
14
+ {%- if messages[0]['content'] is string %}
15
+ {%- set system_message = messages[0]['content']|trim %}
16
+ {%- else %}
17
+ {%- set system_message = messages[0]['content'][0]['text']|trim %}
18
+ {%- endif %}
19
+ {%- set messages = messages[1:] %}
20
+{%- else %}
21
+ {%- if tools is not none %}
22
+ {#- Add default tool system message when tools are provided #}
23
+ {%- set system_message = "You are a helpful assistant with tool calling "
24
+ "capabilities. Only reply with a tool call if the function exists in the "
25
+ "library provided by the user. If it doesn't exist, just reply directly in "
26
+ "natural language. When you receive a tool call response, use the output to "
27
+ "format an answer to the original user question." %}
28
+ {%- else %}
29
+ {%- set system_message = "" %}
30
+ {%- endif %}
31
+{%- endif %}
32
+
33
+{#- System message if the user supplied one, or if tools are used (default tool system message) #}
34
+{%- if system_message %}
35
+ {#- always use user provided system message to override default tool system message #}
36
+ {{- "<|header_start|>system<|header_end|>\n\n" }}
37
+ {{- system_message }}
38
+ {%- if tools is not none and not tools_in_user_message %}
39
+ {{- "Tools: You have access to the following tools. You might need to use one "
40
+ "or more function/tool calls to fulfill the task. \n"
41
+ "If none are needed, then proceed to the response.\n\n"
42
+ "Tool Call Syntax: You can call tools using the following syntax:\n"
43
+ "[func_name1(params_name1=params_value1, params_name2=params_value2, ...), ...]\n"
44
+ "Do not include anything else when calling the tools with the syntax above.\n\n"
45
+ "Here is a list of functions in JSON format that you can invoke.\n " }}
46
+ {%- for t in tools %}
47
+ {{- t | tojson(indent=4) }}
48
+ {{- "\n\n" }}
49
+ {%- endfor %}
50
+ {%- endif %}
51
+ {{- "<|eot|>" }}
52
+{%- endif %}
53
+
54
+{#- Custom tools are passed in a user message with some extra guidance #}
55
+{%- if tools_in_user_message and tools is not none %}
56
+ {#- Extract the first user message so we can plug it in here #}
57
+ {%- if messages | length != 0 %}
58
+ {%- if messages[0]['content'] is string %}
59
+ {%- set first_user_message = messages[0]['content']|trim %}
60
+ {%- else %}
61
+ {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
62
+ {%- endif %}
63
+ {%- set messages = messages[1:] %}
64
+ {%- else %}
65
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
66
+ {%- endif %}
67
+ {{- '<|header_start|>user<|header_end|>\n\n' -}}
68
+ {{- first_user_message}}
69
+ {{- "\nHere is a list of functions in JSON format that you can invoke:"}}
70
+ {%- for t in tools %}
71
+ {{- t | tojson(indent=4) }}
72
+ {{- "\n\n" }}
73
+ {%- endfor %}
74
+ {{- "Should you decide to return the function call(s), put them in the format "
75
+ "of [func_name1(params_name1=params_value1, params_name2=params_value2, "
76
+ "...), ...]\nDo not include anything else when calling the tools with the "
77
+ "syntax above." }}
78
+{%- endif %}
79
+
80
+{%- for message in messages %}
81
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
82
+ {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
83
+ {%- if message['content'] is string %}
84
+ {{- message['content'] }}
85
+ {%- else %}
86
+ {%- for content in message['content'] %}
87
+ {%- if content['type'] == 'image' %}
88
+ {{- '<|image|>' }}
89
+ {%- elif content['type'] == 'text' %}
90
+ {{- content['text'] | trim }}
91
+ {%- endif %}
92
+ {%- endfor %}
93
+ {%- endif %}
94
+ {{- "<|eot|>" }}
95
+ {%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
96
+ {%- set tool_call = message.tool_calls[0].function %}
97
+ {{- '<|header_start|>assistant<|header_end|>\n\n' -}}
98
+ {%- if message['content'] is string %}
99
+ {{- message['content'] }}
100
+ {%- else %}
101
+ {%- for content in message['content'] %}
102
+ {%- if content['type'] == 'image' %}
103
+ {{- '<|image|>' }}
104
+ {%- elif content['type'] == 'text' %}
105
+ {{- content['text'] }}
106
+ {%- endif %}
107
+ {%- endfor %}
108
+ {%- endif %}
109
+ {%- for tool_call in message.tool_calls %}
110
+ {%- if tool_call.function is defined %}
111
+ {%- set tool_call = tool_call.function %}
112
+ {%- endif %}
113
+ {{- tool_call.name + '(' -}}
114
+ {%- for param in tool_call.arguments %}
115
+ {{- param + '=' -}}
116
+ {{- "%s" | format(tool_call.arguments[param]) -}}
117
+ {% if not loop.last %}, {% endif %}
118
+ {%- endfor %}
119
+ {{- ')' -}}
120
+ {% if not loop.last %}, {% endif %}
121
+ {%- endfor %}
122
+ {{- "<|eom|>" }}
123
+ {%- elif message.role == "tool" or message.role == "ipython" %}
124
+ {{- "<|header_start|>ipython<|header_end|>\n\n" }}
125
+ {%- if message.content is string %}
126
+ {{- message.content | tojson }}
127
+ {%- else %}
128
+ {%- for content in message['content'] %}
129
+ {%- if content['type'] == 'text' %}
130
+ {{- content['text'] | tojson }}
131
+ {%- endif %}
132
+ {%- endfor %}
133
+ {%- endif %}
134
+ {{- "<|eom|>" }}
135
+ {%- endif %}
136
+{%- endfor %}
137
+{%- if add_generation_prompt %}
138
+ {{- '<|header_start|>assistant<|header_end|>\n\n' }}
139
+{%- endif %}
tests/tool_use/conftest.py
+24
-1
@@ -10,10 +10,33 @@ from vllm.platforms import current_platform
10
from .utils import ARGS, CONFIGS, ServerConfig
11
12
13
+# select models to test based on command line arguments
14
+def pytest_addoption(parser):
15
+ parser.addoption("--models",
16
+ nargs="+",
17
+ help="Specify one or more models to test")
18
+ parser.addoption("--extended",
19
+ action="store_true",
20
+ default=False,
21
+ help="invoke extended tests requiring large GPUs")
22
+
23
+
24
# for each server config, download the model and return the config
25
@pytest.fixture(scope="session", params=CONFIGS.keys())
26
def server_config(request):
16
- config = CONFIGS[request.param]
27
+ extended = request.config.getoption("--extended")
28
+ models = request.config.getoption("--models")
29
+
30
+ config_keys_to_test = [
31
+ key for key in CONFIGS if (models is None or key in models) and (
32
+ extended or not CONFIGS[key].get("extended", False))
33
+ ]
34
+
35
+ config_key = request.param
36
+ if config_key not in config_keys_to_test:
37
+ pytest.skip(f"Skipping config '{config_key}'")
38
+
39
+ config = CONFIGS[config_key]
40
41
if current_platform.is_rocm() and not config.get("supports_rocm", True):
42
pytest.skip("The {} model can't be tested on the ROCm platform".format(
tests/tool_use/utils.py
+16
@@ -16,6 +16,7 @@ class ServerConfig(TypedDict, total=False):
16
system_prompt: Optional[str]
17
supports_parallel: Optional[bool]
18
supports_rocm: Optional[bool]
19
+ extended: Optional[bool] # tests do not run in CI automatically
20
21
22
def patch_system_prompt(messages: list[dict[str, Any]],
@@ -82,6 +83,21 @@ CONFIGS: dict[str, ServerConfig] = {
83
"supports_parallel":
84
False,
85
},
86
+ "llama4": {
87
+ "model":
88
+ "meta-llama/Llama-4-Scout-17B-16E-Instruct",
89
+ "arguments": [
90
+ "--enforce-eager", "--no-enable-prefix-caching",
91
+ "--tool-call-parser", "pythonic", "--chat-template",
92
+ str(VLLM_PATH /
93
+ "examples/tool_chat_template_llama4_pythonic.jinja"), "-tp",
94
+ "4"
95
+ ],
96
+ "supports_parallel":
97
+ False,
98
+ "extended":
99
+ True
100
+ },
101
"mistral": {
102
"model":
103
"mistralai/Mistral-7B-Instruct-v0.3",
vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py
+1
-1
@@ -28,7 +28,7 @@ class _UnexpectedAstError(Exception):
28
class PythonicToolParser(ToolParser):
29
"""
30
Tool call parser for models that produce tool calls in a pythonic style,
31
- such as Llama 3.2 models.
31
+ such as Llama 3.2 and Llama 4 models.
32
33
Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set
34
"""