| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """GLM-4.7 parser for reasoning and tool calls. |
| 4 | |
| 5 | GLM-4.7 uses XML-like tool calls:: |
| 6 | |
| 7 | <tool_call>func_name<arg_key>key</arg_key><arg_value>value</arg_value></tool_call> |
| 8 | |
| 9 | The function name can be followed directly by the first ``<arg_key>`` tag, |
| 10 | and tool calls may have no arguments. |
| 11 | """ |
| 12 | |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import functools |
| 16 | import json |
| 17 | from typing import TYPE_CHECKING |
| 18 | |
| 19 | import regex as re |
| 20 | |
| 21 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 22 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 23 | from vllm.logger import init_logger |
| 24 | from vllm.parser.engine.events import EventType |
| 25 | from vllm.parser.engine.parser_engine import ParserEngine |
| 26 | from vllm.parser.engine.parser_engine_config import ( |
| 27 | ParserEngineConfig, |
| 28 | ParserState, |
| 29 | Transition, |
| 30 | ) |
| 31 | |
| 32 | logger = init_logger(__name__) |
| 33 | |
| 34 | if TYPE_CHECKING: |
| 35 | from vllm.tokenizers import TokenizerLike |
| 36 | from vllm.tool_parsers.abstract_tool_parser import Tool |
| 37 | |
| 38 | THINK_START = "<think>" |
| 39 | THINK_END = "</think>" |
| 40 | TOOL_CALL_START = "<tool_call>" |
| 41 | TOOL_CALL_END = "</tool_call>" |
| 42 | ARG_KEY_START = "<arg_key>" |
| 43 | ARG_KEY_END = "</arg_key>" |
| 44 | ARG_VALUE_START = "<arg_value>" |
| 45 | ARG_VALUE_END = "</arg_value>" |
| 46 | |
| 47 | # Special tokens that delimit conversation turns in a rendered GLM prompt. |
| 48 | # Reasoning markers belonging to earlier turns must not be mistaken for the |
| 49 | # state of the turn currently being generated. |
| 50 | GLM_TURN_BOUNDARIES = frozenset( |
| 51 | ("<|system|>", "<|user|>", "<|assistant|>", "<|observation|>") |
| 52 | ) |
| 53 | |
| 54 | _ARG_RE = re.compile( |
| 55 | r"<arg_key>(?P<key>.*?)</arg_key>\s*" |
| 56 | r"<arg_value>(?P<value>.*?)</arg_value>", |
| 57 | re.DOTALL, |
| 58 | ) |
| 59 | _PARTIAL_ARG_RE = re.compile( |
| 60 | r"<arg_key>(?P<key>.*?)</arg_key>\s*" |
| 61 | r"<arg_value>(?P<value>.*)$", |
| 62 | re.DOTALL, |
| 63 | ) |
| 64 | |
| 65 | |
| 66 | def _glm47_arg_converter(raw_args: str, partial: bool) -> str: |
| 67 | params: dict[str, object] = {} |
| 68 | |
| 69 | for match in _ARG_RE.finditer(raw_args): |
| 70 | params[match.group("key").strip()] = match.group("value") |
| 71 | |
| 72 | if partial: |
| 73 | remaining = _ARG_RE.sub("", raw_args) |
| 74 | match = _PARTIAL_ARG_RE.search(remaining) |
| 75 | if match: |
| 76 | key = match.group("key").strip() |
| 77 | if key: |
| 78 | params[key] = match.group("value") |
| 79 | |
| 80 | return json.dumps(params, ensure_ascii=False) |
| 81 | |
| 82 | |
| 83 | def _glm53_always_thinks(tokenizer: TokenizerLike) -> bool: |
| 84 | # GLM-5.3 keeps the GLM-4.x prompt and tool-call format but replaces the |
| 85 | # enable_thinking toggle with an always-on "Reasoning Effort:" header, so |
| 86 | # its generation prompt always opens <think> and the model output carries |
| 87 | # no opening tag. |
| 88 | template = getattr(tokenizer, "chat_template", None) |
| 89 | return ( |
| 90 | isinstance(template, str) |
| 91 | and "[gMASK]<sop>" in template |
| 92 | and "Reasoning Effort:" in template |
| 93 | and "enable_thinking" not in template |
| 94 | and "<tool_call>" in template |
| 95 | and "<arg_key>" in template |
| 96 | and "<arg_value>" in template |
| 97 | ) |
| 98 | |
| 99 | |
| 100 | @functools.cache |
| 101 | def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: |
| 102 | arg_tag_transitions = { |
| 103 | (ParserState.TOOL_ARGS, terminal): Transition( |
| 104 | ParserState.TOOL_ARGS, |
| 105 | (EventType.ARG_VALUE_CHUNK,), |
| 106 | ) |
| 107 | for terminal in ( |
| 108 | "ARG_KEY_START", |
| 109 | "ARG_KEY_END", |
| 110 | "ARG_VALUE_START", |
| 111 | "ARG_VALUE_END", |
| 112 | ) |
| 113 | } |
| 114 | |
| 115 | reasoning_terminals = ( |
| 116 | { |
| 117 | "THINK_START": THINK_START, |
| 118 | "THINK_END": THINK_END, |
| 119 | } |
| 120 | if thinking |
| 121 | else {} |
| 122 | ) |
| 123 | reasoning_token_id_terminals = ( |
| 124 | { |
| 125 | "THINK_START": THINK_START, |
| 126 | "THINK_END": THINK_END, |
| 127 | } |
| 128 | if thinking |
| 129 | else {} |
| 130 | ) |
| 131 | reasoning_transitions = ( |
| 132 | { |
| 133 | (ParserState.CONTENT, "THINK_START"): Transition( |
| 134 | ParserState.REASONING, |
| 135 | (EventType.REASONING_START,), |
| 136 | ), |
| 137 | (ParserState.REASONING, "THINK_END"): Transition( |
| 138 | ParserState.CONTENT, |
| 139 | (EventType.REASONING_END,), |
| 140 | ), |
| 141 | (ParserState.CONTENT, "THINK_END"): Transition( |
| 142 | ParserState.CONTENT, |
| 143 | (), |
| 144 | ), |
| 145 | } |
| 146 | if thinking |
| 147 | else {} |
| 148 | ) |
| 149 | |
| 150 | return ParserEngineConfig( |
| 151 | name="glm47_moe", |
| 152 | initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, |
| 153 | terminals={ |
| 154 | **reasoning_terminals, |
| 155 | "TOOL_START": TOOL_CALL_START, |
| 156 | "TOOL_END": TOOL_CALL_END, |
| 157 | "ARG_KEY_START": ARG_KEY_START, |
| 158 | "ARG_KEY_END": ARG_KEY_END, |
| 159 | "ARG_VALUE_START": ARG_VALUE_START, |
| 160 | "ARG_VALUE_END": ARG_VALUE_END, |
| 161 | }, |
| 162 | token_id_terminals={ |
| 163 | **reasoning_token_id_terminals, |
| 164 | "TOOL_START": TOOL_CALL_START, |
| 165 | "TOOL_END": TOOL_CALL_END, |
| 166 | }, |
| 167 | transitions={ |
| 168 | **reasoning_transitions, |
| 169 | (ParserState.REASONING, "THINK_START"): Transition( |
| 170 | ParserState.REASONING, |
| 171 | (), |
| 172 | ), |
| 173 | (ParserState.REASONING, "TOOL_START"): Transition( |
| 174 | ParserState.TOOL_NAME, |
| 175 | (EventType.REASONING_END, EventType.TOOL_CALL_START), |
| 176 | ), |
| 177 | (ParserState.CONTENT, "TOOL_START"): Transition( |
| 178 | ParserState.TOOL_NAME, |
| 179 | (EventType.TOOL_CALL_START,), |
| 180 | ), |
| 181 | (ParserState.TOOL_NAME, "ARG_KEY_START"): Transition( |
| 182 | ParserState.TOOL_ARGS, |
| 183 | (EventType.ARG_VALUE_CHUNK,), |
| 184 | ), |
| 185 | (ParserState.TOOL_NAME, "TOOL_END"): Transition( |
| 186 | ParserState.CONTENT, |
| 187 | (EventType.TOOL_CALL_END,), |
| 188 | ), |
| 189 | (ParserState.TOOL_ARGS, "TOOL_END"): Transition( |
| 190 | ParserState.CONTENT, |
| 191 | (EventType.TOOL_CALL_END,), |
| 192 | ), |
| 193 | **arg_tag_transitions, |
| 194 | }, |
| 195 | turn_boundary_tokens=GLM_TURN_BOUNDARIES, |
| 196 | arg_converter=_glm47_arg_converter, |
| 197 | stream_arg_deltas=True, |
| 198 | tool_args_json=False, |
| 199 | validate_tool_names=True, |
| 200 | ) |
| 201 | |
| 202 | |
| 203 | class Glm47MoeParser(ParserEngine): |
| 204 | """GLM-4.7 parser backed by the declarative parser engine.""" |
| 205 | |
| 206 | def __init__( |
| 207 | self, |
| 208 | tokenizer: TokenizerLike, |
| 209 | tools: list[Tool] | None = None, |
| 210 | **kwargs, |
| 211 | ) -> None: |
| 212 | chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} |
| 213 | thinking = chat_kwargs.get("thinking", None) |
| 214 | enable_thinking = chat_kwargs.get("enable_thinking", None) |
| 215 | if (thinking is False or enable_thinking is False) and _glm53_always_thinks( |
| 216 | tokenizer |
| 217 | ): |
| 218 | logger.warning_once( |
| 219 | "Ignoring enable_thinking/thinking: the GLM-5.3 chat template " |
| 220 | "has no thinking switch, so reasoning is always on and " |
| 221 | "disabling extraction would leak it into the content." |
| 222 | ) |
| 223 | thinking = None |
| 224 | enable_thinking = None |
| 225 | self.thinking_enabled = ( |
| 226 | True |
| 227 | if thinking is None and enable_thinking is None |
| 228 | else bool(thinking) or bool(enable_thinking) |
| 229 | ) |
| 230 | kwargs.setdefault( |
| 231 | "parser_engine_config", |
| 232 | glm47_moe_config(thinking=self.thinking_enabled), |
| 233 | ) |
| 234 | super().__init__(tokenizer, tools, **kwargs) |
| 235 | |
| 236 | def _emit_name_delta(self, idx: int, deltas, name: str | None) -> None: |
| 237 | if name is not None: |
| 238 | name = name.strip() |
| 239 | super()._emit_name_delta(idx, deltas, name) |
| 240 | |
| 241 | def _handle_tool_end(self, event, deltas) -> None: |
| 242 | idx = event.tool_index |
| 243 | if 0 <= idx < len(self._tool_slots): |
| 244 | self._tool_slots[idx].name = self._tool_slots[idx].name.strip() |
| 245 | super()._handle_tool_end(event, deltas) |
| 246 | |
| 247 | def extract_reasoning( |
| 248 | self, |
| 249 | model_output: str, |
| 250 | request: ChatCompletionRequest | ResponsesRequest, |
| 251 | ) -> tuple[str | None, str | None]: |
| 252 | if not self.thinking_enabled: |
| 253 | return None, model_output |
| 254 | return super().extract_reasoning(model_output, request) |