A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Add chat template for Llama 4 models (#16428)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Apr 24, 2025 at 17:19 UTC
05e1fbfc52ca575e6539de63dbb5fab929683162
4 files changed
+139
-1
docs/source/features/tool_calling.md
+8
-1
@@ -152,10 +152,11 @@ Recommended flags: `--tool-call-parser mistral --chat-template examples/tool_cha
152
153
Supported models:
154
155
-All Llama 3.1 and 3.2 models should be supported.
155
+All Llama 3.1, 3.2 and 4 models should be supported.
156
157
* `meta-llama/Llama-3.1-*`
158
* `meta-llama/Llama-3.2-*`
159
+* `meta-llama/Llama-4-*`
160
161
The tool calling that is supported is the [JSON based tool calling](https://llama.meta.com/docs/model-cards-and-prompt-formats/llama3_1/#json-based-tool-calling). For [pythonic tool calling](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#zero-shot-function-calling) introduced by the Llama-3.2 models, see the `pythonic` tool parser below.
162
@@ -176,6 +177,12 @@ images.
177
178
Recommended flags: `--tool-call-parser llama3_json --chat-template {see_above}`
179
180
+VLLM also provides a JSON based chat template for Llama 4:
181
+* `examples/tool_chat_template_llama4_json.jinja` - this is based on the "official" chat template for the Llama 4
182
+models, but tweaked so that it works better with vLLM.
183
+
184
+For Llama 4 use `--tool-call-parser llama4_json examples/tool_chat_template_llama4_json.jinja`.
185
+
186
#### IBM Granite
187
188
Supported models:
examples/tool_chat_template_llama4_json.jinja
new
+116
@@ -0,0 +1,116 @@
1
+{%- macro is_array_of_type_objects(var) -%}
2
+ {%- if var is iterable and var is not string -%}
3
+ {%- set valid = true -%}
4
+ {%- for item in var -%}
5
+ {%- if 'type' not in item -%}
6
+ {%- set valid = false -%}
7
+ {%- break -%}
8
+ {%- endif -%}
9
+ {%- endfor -%}
10
+ {{ valid }}
11
+ {%- else -%}
12
+ {{ false }}
13
+ {%- endif -%}
14
+{%- endmacro %}
15
+
16
+{%- macro render_message(message) %}
17
+ {%- if message['content'] is string %}
18
+ {{- message['content']|trim }}
19
+ {%- elif is_array_of_type_objects(data) == 'True' %}
20
+ {%- for content in message['content'] %}
21
+ {%- if content['type'] == 'image' %}
22
+ {{- '<|image|>' }}
23
+ {%- elif content['type'] == 'text' %}
24
+ {{- content['text']|trim }}
25
+ {%- endif %}
26
+ {%- endfor %}
27
+ {%- else %}
28
+ {{- message['content']|tojson }}
29
+ {%- endif %}
30
+{%- endmacro %}
31
+
32
+{{- bos_token }}
33
+{%- if custom_tools is defined %}
34
+ {%- set tools = custom_tools %}
35
+{%- endif %}
36
+{%- if not tools_in_user_message is defined %}
37
+ {%- set tools_in_user_message = true %}
38
+{%- endif %}
39
+{%- if not tools is defined %}
40
+ {%- set tools = none %}
41
+{%- endif %}
42
+
43
+{#- This block extracts the system message, so we can slot it into the right place. #}
44
+{%- if messages[0]['role'] == 'system' %}
45
+ {%- set system_message = messages[0] %}
46
+ {%- set messages = messages[1:] %}
47
+{%- else %}
48
+ {%- set system_message = ({ "content": "You are a helpful assistant with tool calling "
49
+ "capabilities. Only reply with a tool call if the function exists in the "
50
+ "library provided by the user. If it doesn't exist, just reply directly in "
51
+ "natural language. When you receive a tool call response, use the output to "
52
+ "format an answer to the original user question."}) %}
53
+{%- endif %}
54
+
55
+{%- set tool_lib_preamble = 'Tools: You have access to the following tools. You might need to use one '
56
+ 'or more function/tool calls to fulfill the task. \n'
57
+ 'If none are needed, then proceed to the response.\n\n'
58
+ 'Tool Call Syntax: You can call tools using the following syntax:\n'
59
+ '{"name": function name, "parameters": dictionary of argument name and its value}.\n'
60
+ 'Separate multiple function calls by "; ". Do not use variables.\n'
61
+ 'Do not include anything else when calling the tools with the syntax above.\n\n'
62
+ 'Here is a list of functions in JSON format that you can invoke.\n' %}
63
+
64
+{{- "<|header_start|>system<|header_end|>\n\n" }}
65
+{%- if tools is not none and not tools_in_user_message %}
66
+ {{- tool_lib_preamble }}
67
+ {%- for t in tools %}
68
+ {{- t | tojson(indent=4) }}
69
+ {{- "\n\n" }}
70
+ {%- endfor %}
71
+{%- endif %}
72
+{{- render_message(system_message) }}
73
+{{ "<|eot|>\n" }}
74
+
75
+{#- Custom tools are passed in a user message with some extra guidance #}
76
+{%- if tools_in_user_message and not tools is none %}
77
+ {#- Extract the first user message so we can plug it in here #}
78
+ {%- if messages | length != 0 %}
79
+ {%- set first_user_message = messages[0] %}
80
+ {%- set messages = messages[1:] %}
81
+ {%- else %}
82
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
83
+ {%- endif %}
84
+ {{- '<|header_start|>user<|header_end|>\n\n' }}
85
+ {{- tool_lib_preamble }}
86
+ {%- for t in tools %}
87
+ {{- t | tojson(indent=4) }}
88
+ {{- "\n\n" }}
89
+ {%- endfor %}
90
+ {{- render_message(first_user_message) + "\n<|eot|>"}}
91
+{%- endif %}
92
+
93
+{%- for message in messages %}
94
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
95
+ {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
96
+ {{- render_message(message) }}
97
+ {{- "\n<|eot|>" }}
98
+ {%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
99
+ {{- '\n<|header_start|>assistant<|header_end|>\n\n' -}}
100
+ {{- render_message(message) }}
101
+ {%- for tool_call in message.tool_calls %}
102
+ {{- '{"name": "' + tool_call.function.name + '", ' }}
103
+ {{- '"parameters": ' }}
104
+ {{- tool_call.function.arguments | tojson }}
105
+ {{- "}" }}
106
+ {%- endfor %}
107
+ {{- "\n<|eot|>" }}
108
+ {%- elif message.role == "tool" or message.role == "ipython" %}
109
+ {{- "\n<|header_start|>ipython<|header_end|>\n\n" }}
110
+ {{- render_message(message) }}
111
+ {{- "\n<|eom|>" }}
112
+ {%- endif %}
113
+{%- endfor %}
114
+{%- if add_generation_prompt %}
115
+ {{- '\n<|header_start|>assistant<|header_end|>\n\n' }}
116
+{%- endif %}
tests/tool_use/utils.py
+14
@@ -98,6 +98,20 @@ CONFIGS: dict[str, ServerConfig] = {
98
"extended":
99
True
100
},
101
+ "llama4_json": {
102
+ "model":
103
+ "meta-llama/Llama-4-Scout-17B-16E-Instruct",
104
+ "arguments": [
105
+ "--enforce-eager", "--no-enable-prefix-caching", "-tp", "4",
106
+ "--distributed-executor-backend", "mp", "--tool-call-parser",
107
+ "llama4_json", "--chat-template",
108
+ str(VLLM_PATH / "examples/tool_chat_template_llama4_json.jinja")
109
+ ],
110
+ "supports_parallel":
111
+ True,
112
+ "extended":
113
+ True
114
+ },
115
"mistral": {
116
"model":
117
"mistralai/Mistral-7B-Instruct-v0.3",
vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py
+1
@@ -27,6 +27,7 @@ logger = init_logger(__name__)
27
28
29
@ToolParserManager.register_module("llama3_json")
30
+@ToolParserManager.register_module("llama4_json")
31
class Llama3JsonToolParser(ToolParser):
32
"""
33
Tool call parser for Llama 3.1 models intended for use with the