A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix][Frontend] Update Llama Chat Templates to also support Non-Tool use (#10164)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Nov 22, 2024 at 19:17 UTC
9195dbdbcadb681db67181a664521bd6ef98deee
3 files changed
+110
-36
examples/tool_chat_template_llama3.1_json.jinja
+36
-10
@@ -19,10 +19,18 @@
19
20
{#- This block extracts the system message, so we can slot it into the right place. #}
21
{%- if messages[0]['role'] == 'system' %}
22
- {%- set system_message = messages[0]['content']|trim %}
22
+ {%- if messages[0]['content'] is string %}
23
+ {%- set system_message = messages[0]['content']|trim %}
24
+ {%- else %}
25
+ {%- set system_message = messages[0]['content'][0]['text']|trim %}
26
+ {%- endif %}
27
{%- set messages = messages[1:] %}
28
{%- else %}
25
- {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
29
+ {%- if tools is not none %}
30
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
31
+ {%- else %}
32
+ {%- set system_message = "" %}
33
+ {%- endif %}
34
{%- endif %}
35
36
{#- System message #}
@@ -33,8 +41,8 @@
41
{{- "Cutting Knowledge Date: December 2023\n" }}
42
{{- "Today Date: " + date_string + "\n\n" }}
43
{%- if tools is not none and not tools_in_user_message %}
36
- {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
37
- {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
44
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
45
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
46
{{- "Do not use variables.\n\n" }}
47
{%- for t in tools %}
48
{{- t | tojson(indent=4) }}
@@ -48,7 +56,11 @@
56
{%- if tools_in_user_message and not tools is none %}
57
{#- Extract the first user message so we can plug it in here #}
58
{%- if messages | length != 0 %}
51
- {%- set first_user_message = messages[0]['content']|trim %}
59
+ {%- if messages[0]['content'] is string %}
60
+ {%- set first_user_message = messages[0]['content']|trim %}
61
+ {%- else %}
62
+ {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
63
+ {%- endif %}
64
{%- set messages = messages[1:] %}
65
{%- else %}
66
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
@@ -56,7 +68,7 @@
68
{{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
69
{{- "Given the following functions, please respond with a JSON for a function call " }}
70
{{- "with its proper arguments that best answers the given prompt.\n\n" }}
59
- {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
71
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
72
{{- "Do not use variables.\n\n" }}
73
{%- for t in tools %}
74
{{- t | tojson(indent=4) }}
@@ -67,7 +79,17 @@
79
80
{%- for message in messages %}
81
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
70
- {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
82
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
83
+ {%- if message['content'] is string %}
84
+ {{- message['content'] | trim}}
85
+ {%- else %}
86
+ {%- for content in message['content'] %}
87
+ {%- if content['type'] == 'text' %}
88
+ {{- content['text'] | trim }}
89
+ {%- endif %}
90
+ {%- endfor %}
91
+ {%- endif %}
92
+ {{- '<|eot_id|>' }}
93
{%- elif 'tool_calls' in message %}
94
{%- if not message.tool_calls|length == 1 %}
95
{{- raise_exception("This model only supports single tool-calls at once!") }}
@@ -81,10 +103,14 @@
103
{{- "<|eot_id|>" }}
104
{%- elif message.role == "tool" or message.role == "ipython" %}
105
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
84
- {%- if message.content is mapping %}
85
- {{- message.content | tojson }}
86
- {%- else %}
106
+ {%- if message.content is string %}
107
{{- { "output": message.content } | tojson }}
108
+ {%- else %}
109
+ {%- for content in message['content'] %}
110
+ {%- if content['type'] == 'text' %}
111
+ {{- { "output": content['text'] } | tojson }}
112
+ {%- endif %}
113
+ {%- endfor %}
114
{%- endif %}
115
{{- "<|eot_id|>" }}
116
{%- endif %}
examples/tool_chat_template_llama3.2_json.jinja
+72
-24
@@ -16,38 +16,70 @@
16
{%- set tools = none %}
17
{%- endif %}
18
19
+{#- Find out if there are any images #}
20
+{% set image_ns = namespace(has_images=false) %}
21
+{%- for message in messages %}
22
+ {%- for content in message['content'] %}
23
+ {%- if content['type'] == 'image' %}
24
+ {%- set image_ns.has_images = true %}
25
+ {%- endif %}
26
+ {%- endfor %}
27
+{%- endfor %}
28
+
29
+
30
{#- This block extracts the system message, so we can slot it into the right place. #}
31
{%- if messages[0]['role'] == 'system' %}
21
- {%- set system_message = messages[0]['content']|trim %}
32
+ {%- if messages[0]['content'] is string %}
33
+ {%- set system_message = messages[0]['content']|trim %}
34
+ {%- else %}
35
+ {#- Support vLLM's transforming of a content string to JSON. #}
36
+ {%- set system_message = messages[0]['content'][0]['text']|trim %}
37
+ {%- endif %}
38
{%- set messages = messages[1:] %}
39
{%- else %}
24
- {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
40
+ {%- if tools is not none %}
41
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
42
+ {%- else %}
43
+ {%- set system_message = "" %}
44
+ {%- endif %}
45
{%- endif %}
46
27
-{#- System message #}
28
-{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
29
-{%- if tools is not none %}
30
- {{- "Environment: ipython\n" }}
47
+{#- Including an image is not compatible with a system message #}
48
+{%- if image_ns.has_images and not system_message == "" %}
49
+ {{- raise_exception("Prompting with images is incompatible with system messages and tool use.") }}
50
{%- endif %}
32
-{{- "Cutting Knowledge Date: December 2023\n" }}
33
-{{- "Today Date: " + date_string + "\n\n" }}
34
-{%- if tools is not none and not tools_in_user_message %}
35
- {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
36
- {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
37
- {{- "Do not use variables.\n\n" }}
38
- {%- for t in tools %}
39
- {{- t | tojson(indent=4) }}
40
- {{- "\n\n" }}
41
- {%- endfor %}
51
+
52
+
53
+{#- System message, if there are no images #}
54
+{%- if not image_ns.has_images %}
55
+ {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
56
+ {%- if tools is not none %}
57
+ {{- "Environment: ipython\n" }}
58
+ {%- endif %}
59
+ {{- "Cutting Knowledge Date: December 2023\n" }}
60
+ {{- "Today Date: " + date_string + "\n\n" }}
61
+ {%- if tools is not none and not tools_in_user_message %}
62
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
63
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
64
+ {{- "Do not use variables.\n\n" }}
65
+ {%- for t in tools %}
66
+ {{- t | tojson(indent=4) }}
67
+ {{- "\n\n" }}
68
+ {%- endfor %}
69
+ {%- endif %}
70
+ {{- system_message }}
71
+ {{- "<|eot_id|>" }}
72
{%- endif %}
43
-{{- system_message }}
44
-{{- "<|eot_id|>" }}
73
74
{#- Custom tools are passed in a user message with some extra guidance #}
75
{%- if tools_in_user_message and not tools is none %}
76
{#- Extract the first user message so we can plug it in here #}
77
{%- if messages | length != 0 %}
50
- {%- set first_user_message = messages[0]['content']|trim %}
78
+ {%- if messages[0]['content'] is string %}
79
+ {%- set first_user_message = messages[0]['content']|trim %}
80
+ {%- else %}
81
+ {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
82
+ {%- endif %}
83
{%- set messages = messages[1:] %}
84
{%- else %}
85
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
@@ -55,7 +87,7 @@
87
{{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
88
{{- "Given the following functions, please respond with a JSON for a function call " }}
89
{{- "with its proper arguments that best answers the given prompt.\n\n" }}
58
- {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
90
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
91
{{- "Do not use variables.\n\n" }}
92
{%- for t in tools %}
93
{{- t | tojson(indent=4) }}
@@ -66,7 +98,19 @@
98
99
{%- for message in messages %}
100
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
69
- {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
101
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
102
+ {%- if message['content'] is string %}
103
+ {{- message['content'] | trim}}
104
+ {%- else %}
105
+ {%- for content in message['content'] %}
106
+ {%- if content['type'] == 'image' %}
107
+ {{- '<|image|>' }}
108
+ {%- elif content['type'] == 'text' %}
109
+ {{- content['text'] | trim }}
110
+ {%- endif %}
111
+ {%- endfor %}
112
+ {%- endif %}
113
+ {{- '<|eot_id|>' }}
114
{%- elif 'tool_calls' in message %}
115
{%- if not message.tool_calls|length == 1 %}
116
{{- raise_exception("This model only supports single tool-calls at once!") }}
@@ -80,10 +124,14 @@
124
{{- "<|eot_id|>" }}
125
{%- elif message.role == "tool" or message.role == "ipython" %}
126
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
83
- {%- if message.content is mapping %}
84
- {{- message.content | tojson }}
85
- {%- else %}
127
+ {%- if message.content is string %}
128
{{- { "output": message.content } | tojson }}
129
+ {%- else %}
130
+ {%- for content in message['content'] %}
131
+ {%- if content['type'] == 'text' %}
132
+ {{- { "output": content['text'] } | tojson }}
133
+ {%- endif %}
134
+ {%- endfor %}
135
{%- endif %}
136
{{- "<|eot_id|>" }}
137
{%- endif %}
tests/entrypoints/test_chat_utils.py
+2
-2
@@ -766,8 +766,8 @@ def test_resolve_content_format_hf_defined(model, expected_format):
766
("tool_chat_template_granite_20b_fc.jinja", "string"),
767
("tool_chat_template_hermes.jinja", "string"),
768
("tool_chat_template_internlm2_tool.jinja", "string"),
769
- ("tool_chat_template_llama3.1_json.jinja", "string"),
770
- ("tool_chat_template_llama3.2_json.jinja", "string"),
769
+ ("tool_chat_template_llama3.1_json.jinja", "openai"),
770
+ ("tool_chat_template_llama3.2_json.jinja", "openai"),
771
("tool_chat_template_mistral_parallel.jinja", "string"),
772
("tool_chat_template_mistral.jinja", "string")],
773
)