@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9195dbdbca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix][Frontend] Update Llama Chat Templates to also support Non-Tool use (#10164)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Nov 22, 2024 at 19:17 UTC 9195dbdbcadb681db67181a664521bd6ef98deee
3 files changed +110 -36
examples/tool_chat_template_llama3.1_json.jinja
+36 -10
@@ -19,10 +19,18 @@
19
20 {#- This block extracts the system message, so we can slot it into the right place. #}
21 {%- if messages[0]['role'] == 'system' %}
22 - {%- set system_message = messages[0]['content']|trim %}
22 + {%- if messages[0]['content'] is string %}
23 + {%- set system_message = messages[0]['content']|trim %}
24 + {%- else %}
25 + {%- set system_message = messages[0]['content'][0]['text']|trim %}
26 + {%- endif %}
27 {%- set messages = messages[1:] %}
28 {%- else %}
25 - {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
29 + {%- if tools is not none %}
30 + {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
31 + {%- else %}
32 + {%- set system_message = "" %}
33 + {%- endif %}
34 {%- endif %}
35
36 {#- System message #}
@@ -33,8 +41,8 @@
41 {{- "Cutting Knowledge Date: December 2023\n" }}
42 {{- "Today Date: " + date_string + "\n\n" }}
43 {%- if tools is not none and not tools_in_user_message %}
36 - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
37 - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
44 + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
45 + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
46 {{- "Do not use variables.\n\n" }}
47 {%- for t in tools %}
48 {{- t | tojson(indent=4) }}
@@ -48,7 +56,11 @@
56 {%- if tools_in_user_message and not tools is none %}
57 {#- Extract the first user message so we can plug it in here #}
58 {%- if messages | length != 0 %}
51 - {%- set first_user_message = messages[0]['content']|trim %}
59 + {%- if messages[0]['content'] is string %}
60 + {%- set first_user_message = messages[0]['content']|trim %}
61 + {%- else %}
62 + {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
63 + {%- endif %}
64 {%- set messages = messages[1:] %}
65 {%- else %}
66 {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
@@ -56,7 +68,7 @@
68 {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
69 {{- "Given the following functions, please respond with a JSON for a function call " }}
70 {{- "with its proper arguments that best answers the given prompt.\n\n" }}
59 - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
71 + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
72 {{- "Do not use variables.\n\n" }}
73 {%- for t in tools %}
74 {{- t | tojson(indent=4) }}
@@ -67,7 +79,17 @@
79
80 {%- for message in messages %}
81 {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
70 - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
82 + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
83 + {%- if message['content'] is string %}
84 + {{- message['content'] | trim}}
85 + {%- else %}
86 + {%- for content in message['content'] %}
87 + {%- if content['type'] == 'text' %}
88 + {{- content['text'] | trim }}
89 + {%- endif %}
90 + {%- endfor %}
91 + {%- endif %}
92 + {{- '<|eot_id|>' }}
93 {%- elif 'tool_calls' in message %}
94 {%- if not message.tool_calls|length == 1 %}
95 {{- raise_exception("This model only supports single tool-calls at once!") }}
@@ -81,10 +103,14 @@
103 {{- "<|eot_id|>" }}
104 {%- elif message.role == "tool" or message.role == "ipython" %}
105 {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
84 - {%- if message.content is mapping %}
85 - {{- message.content | tojson }}
86 - {%- else %}
106 + {%- if message.content is string %}
107 {{- { "output": message.content } | tojson }}
108 + {%- else %}
109 + {%- for content in message['content'] %}
110 + {%- if content['type'] == 'text' %}
111 + {{- { "output": content['text'] } | tojson }}
112 + {%- endif %}
113 + {%- endfor %}
114 {%- endif %}
115 {{- "<|eot_id|>" }}
116 {%- endif %}
examples/tool_chat_template_llama3.2_json.jinja
+72 -24
@@ -16,38 +16,70 @@
16 {%- set tools = none %}
17 {%- endif %}
18
19 +{#- Find out if there are any images #}
20 +{% set image_ns = namespace(has_images=false) %}
21 +{%- for message in messages %}
22 + {%- for content in message['content'] %}
23 + {%- if content['type'] == 'image' %}
24 + {%- set image_ns.has_images = true %}
25 + {%- endif %}
26 + {%- endfor %}
27 +{%- endfor %}
28 +
29 +
30 {#- This block extracts the system message, so we can slot it into the right place. #}
31 {%- if messages[0]['role'] == 'system' %}
21 - {%- set system_message = messages[0]['content']|trim %}
32 + {%- if messages[0]['content'] is string %}
33 + {%- set system_message = messages[0]['content']|trim %}
34 + {%- else %}
35 + {#- Support vLLM's transforming of a content string to JSON. #}
36 + {%- set system_message = messages[0]['content'][0]['text']|trim %}
37 + {%- endif %}
38 {%- set messages = messages[1:] %}
39 {%- else %}
24 - {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
40 + {%- if tools is not none %}
41 + {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
42 + {%- else %}
43 + {%- set system_message = "" %}
44 + {%- endif %}
45 {%- endif %}
46
27 -{#- System message #}
28 -{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
29 -{%- if tools is not none %}
30 - {{- "Environment: ipython\n" }}
47 +{#- Including an image is not compatible with a system message #}
48 +{%- if image_ns.has_images and not system_message == "" %}
49 + {{- raise_exception("Prompting with images is incompatible with system messages and tool use.") }}
50 {%- endif %}
32 -{{- "Cutting Knowledge Date: December 2023\n" }}
33 -{{- "Today Date: " + date_string + "\n\n" }}
34 -{%- if tools is not none and not tools_in_user_message %}
35 - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
36 - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
37 - {{- "Do not use variables.\n\n" }}
38 - {%- for t in tools %}
39 - {{- t | tojson(indent=4) }}
40 - {{- "\n\n" }}
41 - {%- endfor %}
51 +
52 +
53 +{#- System message, if there are no images #}
54 +{%- if not image_ns.has_images %}
55 + {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
56 + {%- if tools is not none %}
57 + {{- "Environment: ipython\n" }}
58 + {%- endif %}
59 + {{- "Cutting Knowledge Date: December 2023\n" }}
60 + {{- "Today Date: " + date_string + "\n\n" }}
61 + {%- if tools is not none and not tools_in_user_message %}
62 + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
63 + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
64 + {{- "Do not use variables.\n\n" }}
65 + {%- for t in tools %}
66 + {{- t | tojson(indent=4) }}
67 + {{- "\n\n" }}
68 + {%- endfor %}
69 + {%- endif %}
70 + {{- system_message }}
71 + {{- "<|eot_id|>" }}
72 {%- endif %}
43 -{{- system_message }}
44 -{{- "<|eot_id|>" }}
73
74 {#- Custom tools are passed in a user message with some extra guidance #}
75 {%- if tools_in_user_message and not tools is none %}
76 {#- Extract the first user message so we can plug it in here #}
77 {%- if messages | length != 0 %}
50 - {%- set first_user_message = messages[0]['content']|trim %}
78 + {%- if messages[0]['content'] is string %}
79 + {%- set first_user_message = messages[0]['content']|trim %}
80 + {%- else %}
81 + {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
82 + {%- endif %}
83 {%- set messages = messages[1:] %}
84 {%- else %}
85 {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
@@ -55,7 +87,7 @@
87 {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
88 {{- "Given the following functions, please respond with a JSON for a function call " }}
89 {{- "with its proper arguments that best answers the given prompt.\n\n" }}
58 - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
90 + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
91 {{- "Do not use variables.\n\n" }}
92 {%- for t in tools %}
93 {{- t | tojson(indent=4) }}
@@ -66,7 +98,19 @@
98
99 {%- for message in messages %}
100 {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
69 - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
101 + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
102 + {%- if message['content'] is string %}
103 + {{- message['content'] | trim}}
104 + {%- else %}
105 + {%- for content in message['content'] %}
106 + {%- if content['type'] == 'image' %}
107 + {{- '<|image|>' }}
108 + {%- elif content['type'] == 'text' %}
109 + {{- content['text'] | trim }}
110 + {%- endif %}
111 + {%- endfor %}
112 + {%- endif %}
113 + {{- '<|eot_id|>' }}
114 {%- elif 'tool_calls' in message %}
115 {%- if not message.tool_calls|length == 1 %}
116 {{- raise_exception("This model only supports single tool-calls at once!") }}
@@ -80,10 +124,14 @@
124 {{- "<|eot_id|>" }}
125 {%- elif message.role == "tool" or message.role == "ipython" %}
126 {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
83 - {%- if message.content is mapping %}
84 - {{- message.content | tojson }}
85 - {%- else %}
127 + {%- if message.content is string %}
128 {{- { "output": message.content } | tojson }}
129 + {%- else %}
130 + {%- for content in message['content'] %}
131 + {%- if content['type'] == 'text' %}
132 + {{- { "output": content['text'] } | tojson }}
133 + {%- endif %}
134 + {%- endfor %}
135 {%- endif %}
136 {{- "<|eot_id|>" }}
137 {%- endif %}
tests/entrypoints/test_chat_utils.py
+2 -2
@@ -766,8 +766,8 @@ def test_resolve_content_format_hf_defined(model, expected_format):
766 ("tool_chat_template_granite_20b_fc.jinja", "string"),
767 ("tool_chat_template_hermes.jinja", "string"),
768 ("tool_chat_template_internlm2_tool.jinja", "string"),
769 - ("tool_chat_template_llama3.1_json.jinja", "string"),
770 - ("tool_chat_template_llama3.2_json.jinja", "string"),
769 + ("tool_chat_template_llama3.1_json.jinja", "openai"),
770 + ("tool_chat_template_llama3.2_json.jinja", "openai"),
771 ("tool_chat_template_mistral_parallel.jinja", "string"),
772 ("tool_chat_template_mistral.jinja", "string")],
773 )