A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 14:45 UTC
[Misc] Clean up Gemma4 implementation (#38872)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Apr 3, 2026 at 13:47 UTC
550643541956cf913a2346f69af3be89c5c93a6b
3 files changed
+5
-300
vllm/model_executor/models/gemma4_mm.py
+3
-6
@@ -15,7 +15,6 @@ reason about temporal order.
15
"""
16
17
import math
18
-import sys
18
from collections.abc import Iterable, Mapping, Sequence
19
from typing import Annotated, Any, Literal
20
@@ -480,12 +479,10 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
479
val = merged_kwargs.get("images_kwargs", {}).get("max_soft_tokens")
480
481
if val is not None and val not in _SUPPORTED_SOFT_TOKENS:
483
- logger.error(
484
- "Unsupported max_soft_tokens value: %d. Valid values are %s. Exiting.",
485
- val,
486
- _SUPPORTED_SOFT_TOKENS,
482
+ raise ValueError(
483
+ f"Unsupported max_soft_tokens value: {val}. "
484
+ f"Valid values are {_SUPPORTED_SOFT_TOKENS}."
485
)
488
- sys.exit(1)
486
487
mm_data = dict(mm_data)
488
vllm/model_executor/models/gemma4_utils.py
deleted
-292
@@ -1,292 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-# Copyright 2025 Google Inc. HuggingFace Inc. team. All rights reserved.
4
-
5
-"""Gemma4 output parsing utilities for offline inference.
6
-
7
-Standalone functions that parse decoded model text to extract structured
8
-thinking content and tool calls from Gemma4 models. These are pure-Python
9
-utilities with zero heavy dependencies — they work on raw decoded strings
10
-from any inference backend (vLLM, HuggingFace, TGI, etc.).
11
-
12
-Usage with vLLM offline inference::
13
-
14
- from vllm import LLM, SamplingParams
15
- from vllm.model_executor.models.gemma4_utils import (
16
- parse_output,
17
- parse_tool_calls,
18
- )
19
-
20
- llm = LLM(model="google/gemma-4-it")
21
- outputs = llm.generate(prompt, SamplingParams(...))
22
- text = tokenizer.decode(outputs[0].outputs[0].token_ids, skip_special_tokens=False)
23
-
24
- # Extract thinking / answer (works with or without enable_thinking)
25
- result = parse_output(text)
26
- print(result["thinking"]) # chain-of-thought or None
27
- print(result["answer"]) # final answer
28
-
29
- # Extract tool calls
30
- tool_calls = parse_tool_calls(text)
31
- for tc in tool_calls:
32
- print(f"{tc['name']}({tc['arguments']})")
33
-
34
-Ported from ``transformers.models.gemma4.utils_gemma4`` so that vLLM users
35
-do not need a transformers dependency for output parsing.
36
-"""
37
-
38
-import json
39
-
40
-import regex as re
41
-
42
-# ---- Thinking Mode Utility ----
43
-
44
-# Thinking delimiter tokens as they appear in decoded text.
45
-# Gemma4 uses <|channel> (start) and <channel|> (end) as thinking delimiters.
46
-_THINKING_START_TAG = "<|channel>"
47
-_THINKING_END_TAG = "<channel|>"
48
-
49
-# Sentinel tokens that may appear in decoded output.
50
-_TURN_END_TAG = "<turn|>"
51
-
52
-
53
-def parse_thinking_output(text: str) -> dict[str, str | None]:
54
- """Parse decoded Gemma4 model output.
55
-
56
- Use this on **all** Gemma4 output regardless of whether thinking mode
57
- was enabled. It handles three cases:
58
-
59
- 1. **Thinking enabled, tags present** — splits on ``<|channel>``/
60
- ``<channel|>`` to separate chain-of-thought from the answer and
61
- strips the ``thought\\n`` role label.
62
- 2. **Thinking disabled, spurious label** — strips the bare
63
- ``thought\\n`` prefix that some Gemma4 models emit even
64
- without thinking mode.
65
- 3. **Clean output** — returns the text unchanged.
66
-
67
- The answer text is always cleaned of trailing sentinel tokens
68
- (``<turn|>``, ``<eos>``, etc.).
69
-
70
- Args:
71
- text: Decoded model output text (from ``tokenizer.decode(...)``).
72
-
73
- Returns:
74
- A dict with keys:
75
- - ``"thinking"``: The chain-of-thought text, or ``None`` if no
76
- thinking delimiters were found.
77
- - ``"answer"``: The final answer text.
78
-
79
- Example::
80
-
81
- >>> from vllm.model_executor.models.gemma4_utils import parse_thinking_output
82
- >>> output_text = tokenizer.decode(outputs[0], skip_special_tokens=False)
83
- >>> result = parse_thinking_output(output_text)
84
- >>> print(result["thinking"]) # chain-of-thought reasoning or None
85
- >>> print(result["answer"]) # final answer
86
- """
87
- if _THINKING_END_TAG in text:
88
- parts = text.split(_THINKING_END_TAG, 1)
89
- thinking_block = parts[0]
90
- answer = _clean_answer(parts[1])
91
-
92
- # Extract thinking content: strip the start tag if present
93
- if _THINKING_START_TAG in thinking_block:
94
- thinking = thinking_block.split(_THINKING_START_TAG, 1)[1]
95
- else:
96
- thinking = thinking_block
97
-
98
- # Strip the "thought\n" channel role label the model emits inside
99
- # <|channel>thought\n...<channel|> (analogous to "user\n" in
100
- # <|turn>user\n...<turn|>).
101
- thinking = _strip_thought_label(thinking.strip())
102
- thinking = thinking.strip()
103
-
104
- return {"thinking": thinking, "answer": answer}
105
-
106
- # No thinking delimiters found.
107
- # Strip spurious "thought\n" role label that some Gemma4 models sometimes
108
- # emit even without thinking mode enabled, then clean trailing tokens.
109
- answer = _strip_thought_label(text)
110
- answer = _clean_answer(answer)
111
- return {"thinking": None, "answer": answer}
112
-
113
-
114
-def _strip_thought_label(text: str) -> str:
115
- """Strip the spurious ``thought\\n`` label from the start of text.
116
-
117
- Only strips when ``thought`` appears as the very first word followed by
118
- a newline — preserving the word ``thought`` in any other context.
119
- """
120
- if text.startswith("thought\n"):
121
- return text[len("thought\n") :]
122
- return text
123
-
124
-
125
-def _clean_answer(text: str) -> str:
126
- """Clean trailing sentinel tokens from the answer text.
127
-
128
- Strips ``<turn|>``, ``<eos>``, and surrounding whitespace that the
129
- model appends at the end of its response.
130
- """
131
- text = text.strip()
132
- # Strip trailing <turn|> (Gemma4 turn-end marker)
133
- if text.endswith(_TURN_END_TAG):
134
- text = text[: -len(_TURN_END_TAG)].rstrip()
135
- # Strip trailing <eos> if present
136
- if text.endswith("<eos>"):
137
- text = text[:-5].rstrip()
138
- return text
139
-
140
-
141
-# ---- Tool Call Parsing Utility ----
142
-#
143
-# NOTE: For the OpenAI-compatible API server tool parser (streaming +
144
-# non-streaming), see vllm/tool_parsers/gemma4_tool_parser.py.
145
-# This module provides offline inference utilities for direct user import.
146
-
147
-# Tool call delimiter tokens as they appear in decoded text.
148
-# Standard format: <|tool_call>call:name{args}<tool_call|>
149
-_TOOL_CALL_START_TAG = "<|tool_call>"
150
-_TOOL_CALL_END_TAG = "<tool_call|>"
151
-_TOOL_RESPONSE_START_TAG = "<|tool_response>"
152
-
153
-# Gemma4 escape token as it appears in decoded text.
154
-_ESCAPE_TOKEN = '<|"|>'
155
-
156
-
157
-def _parse_tool_arguments(args_str: str) -> dict[str, str]:
158
- """Parse tool call arguments from the Gemma4 compact format.
159
-
160
- Handles the ``key:<|"|>value<|"|>`` format used by Gemma4, with fallback
161
- to heuristic key-value extraction. Also tolerates the slightly different
162
- ``key: "value"`` format (space + plain quotes) that some chat templates
163
- produce.
164
-
165
- Args:
166
- args_str: Raw argument string from inside ``call:name{...}``.
167
-
168
- Returns:
169
- Dictionary of argument name → value.
170
- """
171
- if not args_str or not args_str.strip():
172
- return {}
173
-
174
- # Replace Gemma4 escape tokens with standard quotes.
175
- cleaned = args_str.replace(_ESCAPE_TOKEN, '"')
176
-
177
- # Try JSON parsing first (handles nested values, arrays, etc.).
178
- try:
179
- parsed = json.loads("{" + cleaned + "}")
180
- # Ensure all values are strings for consistency.
181
- return {k: str(v) if not isinstance(v, str) else v for k, v in parsed.items()}
182
- except (json.JSONDecodeError, ValueError):
183
- pass
184
-
185
- # Fallback: extract key:"value" pairs (allow optional space after colon).
186
- arguments = {}
187
- for key, value in re.findall(r'(\w+):\s*"([^"]*)"', cleaned):
188
- arguments[key] = value
189
-
190
- if not arguments:
191
- # Last resort: extract key:value pairs (unquoted).
192
- for key, value in re.findall(r"(\w+):\s*([^,}]+)", args_str):
193
- arguments[key] = value.strip().strip('"').replace(_ESCAPE_TOKEN, "")
194
-
195
- return arguments
196
-
197
-
198
-def parse_tool_calls(text: str, *, strict: bool = False) -> list[dict]:
199
- """Parse tool calls from decoded Gemma4 model output.
200
-
201
- Uses a tiered parsing strategy to handle known output variations in
202
- Gemma4 models, which may emit
203
- non-standard tool call formats.
204
-
205
- Parsing tiers:
206
- 1. **Standard**: ``<|tool_call>call:name{args}<tool_call|>``
207
- (special token IDs 48/49 in decoded text)
208
- 2. **Fallback** (when ``strict=False``): bare ``call:name{args}``
209
- patterns, including ``<call>name{args}`` (fragmented tokens from
210
- multimodal inputs)
211
-
212
- Args:
213
- text: Decoded model output text (from ``tokenizer.decode(...,
214
- skip_special_tokens=False)``).
215
- strict: If ``True``, only match the standard ``<|tool_call>`` format.
216
- If ``False`` (default), also try fallback patterns for
217
- known Gemma4 output variations.
218
-
219
- Returns:
220
- A list of dicts, each with keys:
221
- - ``"name"``: The tool function name (e.g. ``"get_weather"``).
222
- - ``"arguments"``: A dict of argument name → value.
223
-
224
- Example::
225
-
226
- >>> from vllm.model_executor.models.gemma4_utils import (
227
- ... parse_tool_calls
228
- ... )
229
- >>> output = tokenizer.decode(outputs[0], skip_special_tokens=False)
230
- >>> tool_calls = parse_tool_calls(output)
231
- >>> for tc in tool_calls:
232
- ... print(f"Call: {tc['name']}({tc['arguments']})")
233
- """
234
- results = []
235
-
236
- # Tier 1: Standard format with special tokens.
237
- # <|tool_call>call:name{args}<tool_call|>
238
- # Note: Some Gemma4 models emit <turn|> instead of <tool_call|>.
239
- standard_pattern = r"<\|tool_call\>call:(\w+)\{(.*?)\}(?:<tool_call\|>|<turn\|>)"
240
- for match in re.finditer(standard_pattern, text, re.DOTALL):
241
- name, args_str = match.group(1), match.group(2)
242
- results.append(
243
- {
244
- "name": name,
245
- "arguments": _parse_tool_arguments(args_str),
246
- }
247
- )
248
-
249
- if results or strict:
250
- return results
251
-
252
- # Tier 2: Fallback for known Gemma4 output variations.
253
- # Matches: <call>name{args}, call:name{args}, or bare call:name{args}<eos>
254
- fallback_pattern = r"(?:<call>|(?:^|\s)call:)(\w+)\{(.*?)\}"
255
- for match in re.finditer(fallback_pattern, text, re.DOTALL):
256
- name, args_str = match.group(1), match.group(2)
257
- results.append(
258
- {
259
- "name": name,
260
- "arguments": _parse_tool_arguments(args_str),
261
- }
262
- )
263
-
264
- return results
265
-
266
-
267
-def has_tool_response_tag(text: str) -> bool:
268
- """Check if model output properly ends with a tool response tag.
269
-
270
- Some Gemma4 models sometimes emit ``<eos>`` instead of
271
- ``<|tool_response>`` after a tool call. This helper detects
272
- whether the model used the proper termination, so callers can
273
- decide whether to inject ``<|tool_response>`` into the next prompt.
274
-
275
- Args:
276
- text: Decoded model output text.
277
-
278
- Returns:
279
- ``True`` if the output ends with ``<|tool_response>``
280
- (proper behavior), ``False`` otherwise.
281
-
282
- Example::
283
-
284
- >>> from vllm.model_executor.models.gemma4_utils import (
285
- ... has_tool_response_tag
286
- ... )
287
- >>> if not has_tool_response_tag(model_output):
288
- ... # Model used <eos> instead — inject <|tool_response> manually
289
- ... next_prompt = "<|tool_response>" + tool_result
290
- """
291
- stripped = text.rstrip()
292
- return stripped.endswith(_TOOL_RESPONSE_START_TAG)
vllm/transformers_utils/model_arch_config_convertor.py
+2
-2
@@ -469,6 +469,8 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
469
"mpt": MPTModelArchConfigConvertor,
470
"dbrx": DbrxModelArchConfigConvertor,
471
"falcon": FalconModelArchConfigConvertor,
472
+ "gemma4": Gemma4ModelArchConfigConvertor,
473
+ "gemma4_text": Gemma4ModelArchConfigConvertor,
474
"RefinedWeb": FalconModelArchConfigConvertor,
475
"RefinedWebModel": FalconModelArchConfigConvertor,
476
"nemotron-nas": NemotronNasModelArchConfigConvertor,
@@ -481,6 +483,4 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
483
"ernie_mtp": ErnieMTPModelArchConfigConvertor,
484
"pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,
485
"longcat_flash_mtp": LongCatFlashMTPModelArchConfigConvertor,
484
- "gemma4": Gemma4ModelArchConfigConvertor,
485
- "gemma4_text": Gemma4ModelArchConfigConvertor,
486
}