@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 41e95c5247
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix Hermes tool call chat template bug (#8256)

Co-authored-by: Kyle Mistele <kyle@constellate.ai>

Kyle Mistele committed Sep 6, 2024 at 21:49 UTC 41e95c5247c9703c3e11f3b563d8bba70ed31aca
1 file changed +16 -15
examples/tool_chat_template_hermes.jinja
+16 -15
@@ -89,22 +89,23 @@
89 {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
90 {%- elif message.role == "assistant" and message.tool_calls is defined %}
91 {{- '<|im_start|>' + message.role }}
92 - {%- for tool_call in message.tool_calls %}
93 - {{- '\n<tool_call>\n' }}
94 - {%- if tool_call.function is defined %}
95 - {%- set tool_call = tool_call.function %}
96 - {%- endif %}
97 - {{- '{' }}
98 - {{- '"name": "' }}
99 - {{- tool_call.name }}
100 - {{- '"}' }}
92 + {%- for tool_call in message.tool_calls %}
93 + {{- '\n<tool_call>\n' }}
94 + {%- if tool_call.function is defined %}
95 + {%- set tool_call = tool_call.function %}
96 + {%- endif %}
97 + {{- '{' }}
98 + {{- '"name": "' }}
99 + {{- tool_call.name }}
100 + {{- '"' }}
101 + {%- if tool_call.arguments is defined %}
102 {{- ', ' }}
102 - {%- if tool_call.arguments is defined %}
103 - {{- '"arguments": ' }}
104 - {{- tool_call.arguments|tojson }}
105 - {%- endif %}
106 - {{- '\n</tool_call>' }}
107 - {%- endfor %}
103 + {{- '"arguments": ' }}
104 + {{- tool_call.arguments|tojson }}
105 + {%- endif %}
106 + {{- '}' }}
107 + {{- '\n</tool_call>' }}
108 + {%- endfor %}
109 {{- '<|im_end|>\n' }}
110 {%- elif message.role == "tool" %}
111 {%- if loop.previtem and loop.previtem.role != "tool" %}