A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 17:45 UTC
Fix GLM-4.6v flash tool calling in transformers 5.x (#31622)
Signed-off-by: baonudesifeizhai <baonudesifeizhai@gmail.com>
baonudesifeizhai committed
Jan 5, 2026 at 14:32 UTC
02dbb933cb28eb2b0f05dd491f20273d2f84e517
2 files changed
+68
examples/tool_chat_template_glm4.jinja
new
+54
@@ -0,0 +1,54 @@
1
+{%- set counter = namespace(index=0) -%}
2
+{%- if not tools is defined %}
3
+ {%- set tools = none %}
4
+{%- endif %}
5
+
6
+{%- if messages and messages[0]['role'] == 'system' %}
7
+ {%- set system_message = messages[0]['content']|trim %}
8
+ {%- set messages = messages[1:] %}
9
+{%- else %}
10
+ {%- set system_message = "You are a helpful assistant." %}
11
+{%- endif %}
12
+
13
+{%- if tools is not none %}
14
+ {%- set tool_instruction %}
15
+You have access to the following tools. When you need to call a tool, you MUST use the following format:
16
+
17
+<tool_call>function_name
18
+<arg_key>parameter_name</arg_key>
19
+<arg_value>parameter_value</arg_value>
20
+</tool_call>
21
+
22
+Important rules:
23
+- Always wrap tool calls with <tool_call>...</tool_call> tags
24
+- Put the function name on the first line after <tool_call>
25
+- Use <arg_key> and <arg_value> tags for each parameter
26
+- If a parameter value is a string, keep it as-is. If it's a number or boolean, convert it appropriately
27
+- You can make multiple tool calls if needed
28
+- If no tool is suitable, respond with regular text
29
+
30
+Available tools:
31
+{% endset %}
32
+ {{- tool_instruction + "\n\n" }}
33
+ {%- for t in tools %}
34
+ {{- t | tojson(indent=4) }}
35
+ {{- "\n\n" }}
36
+ {%- endfor %}
37
+{%- endif %}
38
+
39
+{%- for message in messages -%}
40
+ {%- if message['role'] == 'user' -%}
41
+ {{- '[Round ' + counter.index|string + ']\n问：' + message['content'] -}}
42
+ {%- set counter.index = counter.index + 1 -%}
43
+ {%- endif -%}
44
+ {%- if message['role'] == 'assistant' -%}
45
+ {{- '\n答：' + message['content'] -}}
46
+ {%- if (loop.last and add_generation_prompt) or not loop.last -%}
47
+ {{- '\n' -}}
48
+ {%- endif -%}
49
+ {%- endif -%}
50
+{%- endfor -%}
51
+
52
+{%- if add_generation_prompt and messages[-1]['role'] != 'assistant' -%}
53
+ {{- '\n答：' -}}
54
+{%- endif -%}
vllm/tool_parsers/glm4_moe_tool_parser.py
+14
@@ -56,6 +56,20 @@ class Glm4MoeModelToolParser(ToolParser):
56
self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
57
self._buffer = ""
58
59
+ def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
60
+ """
61
+ Adjust request parameters to ensure tool call tokens are not skipped
62
+ during tokenizer decoding.
63
+ """
64
+ request = super().adjust_request(request)
65
+ if request.tools and request.tool_choice != "none":
66
+ # Ensure tool call tokens (<tool_call>, </tool_call>) are not skipped
67
+ # during decoding. Even though they are not marked as special tokens,
68
+ # setting skip_special_tokens=False ensures proper handling in
69
+ # transformers 5.x where decoding behavior may have changed.
70
+ request.skip_special_tokens = False
71
+ return request
72
+
73
def extract_tool_calls(
74
self,
75
model_output: str,