A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 17:30 UTC
[Model] Support Hy3 preview (#40681)
Signed-off-by: stevenkuang <stevenkuang@tencent.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
stevenkuang committed
Apr 23, 2026 at 22:08 UTC
d0009ddb0b96e95bcfae6038e9b8673bd2263058
16 files changed
+2696
docs/models/supported_models.md
+1
@@ -419,6 +419,7 @@ th {
419
| `Grok1ForCausalLM` | Grok2 | `xai-org/grok-2` | ✅︎ | ✅︎ |
420
| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
421
| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
422
+| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ |
423
| `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ |
424
| `InternLMForCausalLM` | InternLM | `internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc. | ✅︎ | ✅︎ |
425
| `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ |
tests/models/registry.py
+5
@@ -324,6 +324,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
324
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
325
"tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
326
),
327
+ "HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True),
328
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
329
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
330
trust_remote_code=True,
@@ -1516,6 +1517,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
1517
is_available_online=False,
1518
min_transformers_version="5.1.0",
1519
),
1520
+ "HYV3MTPModel": _HfExamplesInfo(
1521
+ "tencent/Hy3-preview",
1522
+ speculative_model="tencent/Hy3-preview",
1523
+ ),
1524
"LongCatFlashMTPModel": _HfExamplesInfo(
1525
"meituan-longcat/LongCat-Flash-Chat",
1526
trust_remote_code=True,
tests/reasoning/test_hy_v3_reasoning_parser.py
new
+243
@@ -0,0 +1,243 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import pytest
4
+
5
+from tests.reasoning.utils import run_reasoning_extraction
6
+from vllm.reasoning import ReasoningParser, ReasoningParserManager
7
+from vllm.tokenizers import get_tokenizer
8
+
9
+parser_name = "hy_v3"
10
+MODEL = "tencent/Hy3-preview"
11
+
12
+
13
+@pytest.fixture(scope="module")
14
+def hy_v3_tokenizer():
15
+ return get_tokenizer(tokenizer_name=MODEL)
16
+
17
+
18
+WITH_THINK = {
19
+ "output": "This is a reasoning section</think>This is the rest",
20
+ "reasoning": "This is a reasoning section",
21
+ "content": "This is the rest",
22
+ "is_reasoning_end": True,
23
+ "reasoning_effort": "high",
24
+}
25
+
26
+WITH_THINK_STREAM = {
27
+ "output": "This is a reasoning section</think>This is the rest",
28
+ "reasoning": "This is a reasoning section",
29
+ "content": "This is the rest",
30
+ "is_reasoning_end": True,
31
+ "reasoning_effort": "high",
32
+}
33
+
34
+WITHOUT_THINK = {
35
+ "output": "This is the rest",
36
+ "reasoning": None,
37
+ "content": "This is the rest",
38
+ "is_reasoning_end": True,
39
+ "reasoning_effort": "no_think",
40
+}
41
+
42
+WITHOUT_THINK_STREAM = {
43
+ "output": "This is the rest",
44
+ "reasoning": None,
45
+ "content": "This is the rest",
46
+ "is_reasoning_end": True,
47
+ "reasoning_effort": "no_think",
48
+}
49
+
50
+WITH_REASONING_EFFORT_NONE = {
51
+ "output": "This is the rest",
52
+ "reasoning": None,
53
+ "content": "This is the rest",
54
+ "is_reasoning_end": True,
55
+}
56
+
57
+WITH_REASONING_EFFORT_NONE_STREAM = {
58
+ "output": "This is the rest",
59
+ "reasoning": None,
60
+ "content": "This is the rest",
61
+ "is_reasoning_end": True,
62
+}
63
+
64
+COMPLETE_REASONING = {
65
+ "output": "This is a reasoning section</think>",
66
+ "reasoning": "This is a reasoning section",
67
+ "content": None,
68
+ "is_reasoning_end": True,
69
+ "reasoning_effort": "high",
70
+}
71
+MULTILINE_REASONING = {
72
+ "output": "This is a reasoning\nsection</think>This is the rest\nThat",
73
+ "reasoning": "This is a reasoning\nsection",
74
+ "content": "This is the rest\nThat",
75
+ "is_reasoning_end": True,
76
+ "reasoning_effort": "high",
77
+}
78
+ONLY_OPEN_TAG = {
79
+ "output": "This is a reasoning section",
80
+ "reasoning": "This is a reasoning section",
81
+ "content": None,
82
+ "is_reasoning_end": False,
83
+ "reasoning_effort": "high",
84
+}
85
+
86
+ONLY_OPEN_TAG_STREAM = {
87
+ "output": "This is a reasoning section",
88
+ "reasoning": "This is a reasoning section",
89
+ "content": None,
90
+ "is_reasoning_end": False,
91
+ "reasoning_effort": "high",
92
+}
93
+
94
+TEST_CASES = [
95
+ pytest.param(
96
+ False,
97
+ WITH_THINK,
98
+ id="with_think",
99
+ ),
100
+ pytest.param(
101
+ True,
102
+ WITH_THINK_STREAM,
103
+ id="with_think_stream",
104
+ ),
105
+ pytest.param(
106
+ False,
107
+ WITHOUT_THINK,
108
+ id="without_think",
109
+ ),
110
+ pytest.param(
111
+ True,
112
+ WITHOUT_THINK_STREAM,
113
+ id="without_think_stream",
114
+ ),
115
+ pytest.param(
116
+ False,
117
+ WITH_REASONING_EFFORT_NONE,
118
+ id="with_reasoning_effort_none",
119
+ ),
120
+ pytest.param(
121
+ True,
122
+ WITH_REASONING_EFFORT_NONE_STREAM,
123
+ id="with_reasoning_effort_none_stream",
124
+ ),
125
+ pytest.param(
126
+ False,
127
+ COMPLETE_REASONING,
128
+ id="complete_reasoning",
129
+ ),
130
+ pytest.param(
131
+ True,
132
+ COMPLETE_REASONING,
133
+ id="complete_reasoning_stream",
134
+ ),
135
+ pytest.param(
136
+ False,
137
+ MULTILINE_REASONING,
138
+ id="multiline_reasoning",
139
+ ),
140
+ pytest.param(
141
+ True,
142
+ MULTILINE_REASONING,
143
+ id="multiline_reasoning_stream",
144
+ ),
145
+ pytest.param(
146
+ False,
147
+ ONLY_OPEN_TAG,
148
+ id="only_open_tag",
149
+ ),
150
+ pytest.param(
151
+ True,
152
+ ONLY_OPEN_TAG_STREAM,
153
+ id="only_open_tag_stream",
154
+ ),
155
+]
156
+
157
+STILL_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜>
158
+You are a helpful assistant.
159
+<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜>
160
+What is the capital of France?<｜hy_Assistant｜>
161
+<think>The user is asking for the capital of"""
162
+
163
+DONE_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜>
164
+You are a helpful assistant.
165
+<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜>
166
+What is the capital of France?<｜hy_Assistant｜>
167
+<think>The user is asking for the capital of France.</think>
168
+The capital of France is Paris."""
169
+
170
+MULTI_TURN_STILL_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜>
171
+You are a helpful assistant.
172
+<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜>
173
+What is the capital of France?<｜hy_Assistant｜
174
+><think></think>The capital of France is Paris.<eos:6124c78e>
175
+<｜hy_User｜>What about Chile?<｜hy_Assistant｜>
176
+<think>The user is asking for the capital of"""
177
+
178
+MULTI_TURN_DONE_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜>
179
+You are a helpful assistant.
180
+<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜>
181
+What is the capital of France?<｜hy_Assistant｜
182
+><think></think>The capital of France is Paris.<eos:6124c78e>
183
+<｜hy_User｜>What about Chile?<｜hy_Assistant｜>
184
+<think>The user is asking for the capital of Chile.</think>
185
+The capital of Chile is Santiago."""
186
+
187
+REASONING_END_TEST_CASES = [
188
+ pytest.param(STILL_REASONING_PROMPT, False, id="still_reasoning"),
189
+ pytest.param(DONE_REASONING_PROMPT, True, id="done_reasoning"),
190
+ pytest.param(
191
+ MULTI_TURN_STILL_REASONING_PROMPT, False, id="multi_turn_still_reasoning"
192
+ ),
193
+ pytest.param(
194
+ MULTI_TURN_DONE_REASONING_PROMPT, True, id="multi_turn_done_reasoning"
195
+ ),
196
+]
197
+
198
+
199
+@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
200
+def test_reasoning(
201
+ streaming: bool,
202
+ param_dict: dict,
203
+ hy_v3_tokenizer,
204
+):
205
+ output = hy_v3_tokenizer.tokenize(param_dict["output"])
206
+ output_tokens: list[str] = [
207
+ hy_v3_tokenizer.convert_tokens_to_string([token]) for token in output
208
+ ]
209
+
210
+ parser_kwargs = {}
211
+ if "reasoning_effort" in param_dict:
212
+ parser_kwargs["chat_template_kwargs"] = {
213
+ "reasoning_effort": param_dict["reasoning_effort"]
214
+ }
215
+ parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
216
+ hy_v3_tokenizer,
217
+ **parser_kwargs,
218
+ )
219
+
220
+ reasoning, content = run_reasoning_extraction(
221
+ parser, output_tokens, streaming=streaming
222
+ )
223
+
224
+ assert reasoning == param_dict["reasoning"]
225
+ assert content == param_dict["content"]
226
+
227
+ output_ids = hy_v3_tokenizer.convert_tokens_to_ids(output)
228
+ is_reasoning_end = parser.is_reasoning_end(output_ids)
229
+ assert is_reasoning_end == param_dict["is_reasoning_end"]
230
+
231
+
232
+@pytest.mark.parametrize("prompt, is_reasoning_end", REASONING_END_TEST_CASES)
233
+def test_is_reasoning_end_full_prompt(
234
+ prompt: str, is_reasoning_end: bool, hy_v3_tokenizer
235
+):
236
+ parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
237
+ hy_v3_tokenizer,
238
+ chat_template_kwargs={"reasoning_effort": "high"},
239
+ )
240
+ tokens = hy_v3_tokenizer.tokenize(prompt)
241
+ token_ids = hy_v3_tokenizer.convert_tokens_to_ids(tokens)
242
+ check_is_reasoning_end = parser.is_reasoning_end(token_ids)
243
+ assert check_is_reasoning_end == is_reasoning_end
tests/tool_parsers/test_hy_v3_tool_parser.py
new
+274
@@ -0,0 +1,274 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa: E501
4
+"""Tests for the HYV3 tool call parser."""
5
+
6
+import json
7
+from unittest.mock import Mock
8
+
9
+import pytest
10
+
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
+ ChatCompletionRequest,
13
+ ChatCompletionToolsParam,
14
+ FunctionDefinition,
15
+)
16
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
17
+from vllm.tokenizers import get_tokenizer
18
+from vllm.tool_parsers.hy_v3_tool_parser import HYV3ToolParser
19
+
20
+parser_name = "hy_v3"
21
+MODEL = "tencent/Hy3-preview"
22
+
23
+
24
+@pytest.fixture(scope="module")
25
+def hy_v3_tokenizer():
26
+ return get_tokenizer(tokenizer_name=MODEL)
27
+
28
+
29
+@pytest.fixture
30
+def hy_v3_tool_parser(hy_v3_tokenizer):
31
+ return HYV3ToolParser(hy_v3_tokenizer)
32
+
33
+
34
+@pytest.fixture
35
+def mock_request() -> ChatCompletionRequest:
36
+ request = Mock(spec=ChatCompletionRequest)
37
+ request.tools = [
38
+ ChatCompletionToolsParam(
39
+ function=FunctionDefinition(name="get_current_date", parameters={}),
40
+ ),
41
+ ChatCompletionToolsParam(
42
+ function=FunctionDefinition(
43
+ name="get_weather",
44
+ parameters={
45
+ "type": "object",
46
+ "properties": {
47
+ "city": {"type": "string"},
48
+ "date": {"type": "string"},
49
+ },
50
+ },
51
+ ),
52
+ ),
53
+ ]
54
+ request.tool_choice = "auto"
55
+ return request
56
+
57
+
58
+class TestHYV3ExtractToolCalls:
59
+ def test_no_tool_call(self, hy_v3_tool_parser, mock_request):
60
+ out = "This is a plain response."
61
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
62
+ assert not r.tools_called
63
+ assert r.content == out
64
+
65
+ def test_zero_arg_inline(self, hy_v3_tool_parser, mock_request):
66
+ out = (
67
+ "<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>"
68
+ )
69
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
70
+ assert r.tools_called
71
+ assert r.tool_calls[0].function.name == "get_current_date"
72
+ assert json.loads(r.tool_calls[0].function.arguments) == {}
73
+ assert r.content is None
74
+
75
+ def test_zero_arg_newline(self, hy_v3_tool_parser, mock_request):
76
+ out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
77
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
78
+ assert r.tools_called
79
+ assert r.tool_calls[0].function.name == "get_current_date"
80
+
81
+ def test_args_same_line(self, hy_v3_tool_parser, mock_request):
82
+ out = (
83
+ "<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing"
84
+ "</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call></tool_calls>"
85
+ )
86
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
87
+ assert r.tools_called
88
+ assert json.loads(r.tool_calls[0].function.arguments) == {
89
+ "city": "Beijing",
90
+ "date": "2026-03-30",
91
+ }
92
+
93
+ def test_args_with_newlines(self, hy_v3_tool_parser, mock_request):
94
+ out = (
95
+ "<tool_calls>\n<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Beijing"
96
+ "</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n</tool_calls>"
97
+ )
98
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
99
+ assert r.tools_called
100
+ assert json.loads(r.tool_calls[0].function.arguments) == {
101
+ "city": "Beijing",
102
+ "date": "2026-03-30",
103
+ }
104
+
105
+ def test_content_before(self, hy_v3_tool_parser, mock_request):
106
+ out = "Checking.<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
107
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
108
+ assert r.tools_called
109
+ assert r.content == "Checking."
110
+
111
+ def test_multiple(self, hy_v3_tool_parser, mock_request):
112
+ out = (
113
+ "<tool_calls>\n<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Beijing"
114
+ "</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n"
115
+ "<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Hangzhou</arg_value>\n"
116
+ "<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n</tool_calls>"
117
+ )
118
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
119
+ assert len(r.tool_calls) == 2
120
+
121
+ def test_empty_content_none(self, hy_v3_tool_parser, mock_request):
122
+ out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
123
+ r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
124
+ assert r.content is None
125
+
126
+
127
+def _simulate_streaming(
128
+ parser: HYV3ToolParser,
129
+ deltas: list[str],
130
+ request: ChatCompletionRequest,
131
+) -> list[DeltaMessage | None]:
132
+ results: list[DeltaMessage | None] = []
133
+ previous_text = ""
134
+ previous_token_ids: list[int] = []
135
+ vocab = parser.vocab
136
+ for delta_text in deltas:
137
+ current_text = previous_text + delta_text
138
+ delta_token_ids = [tid for tok, tid in vocab.items() if tok in delta_text]
139
+ current_token_ids = previous_token_ids + delta_token_ids
140
+ result = parser.extract_tool_calls_streaming(
141
+ previous_text=previous_text,
142
+ current_text=current_text,
143
+ delta_text=delta_text,
144
+ previous_token_ids=previous_token_ids,
145
+ current_token_ids=current_token_ids,
146
+ delta_token_ids=delta_token_ids,
147
+ request=request,
148
+ )
149
+ results.append(result)
150
+ previous_text = current_text
151
+ previous_token_ids = current_token_ids
152
+ return results
153
+
154
+
155
+def _collect_streaming_tool_calls(results: list[DeltaMessage | None]) -> list[dict]:
156
+ tool_calls: dict[int, dict] = {}
157
+ for result in results:
158
+ if result is None or not result.tool_calls:
159
+ continue
160
+ for tc in result.tool_calls:
161
+ idx = tc.index
162
+ if idx not in tool_calls:
163
+ tool_calls[idx] = {
164
+ "name": tc.function.name or "",
165
+ "arguments": tc.function.arguments or "",
166
+ }
167
+ else:
168
+ if tc.function.name:
169
+ tool_calls[idx]["name"] += tc.function.name
170
+ if tc.function.arguments:
171
+ tool_calls[idx]["arguments"] += tc.function.arguments
172
+ return [tool_calls[i] for i in sorted(tool_calls.keys())]
173
+
174
+
175
+def _collect_streaming_content(results: list[DeltaMessage | None]) -> str:
176
+ parts = []
177
+ for result in results:
178
+ if result is not None and result.content:
179
+ parts.append(result.content)
180
+ return "".join(parts)
181
+
182
+
183
+class TestHYV3ExtractToolCallsStreaming:
184
+ def test_no_tool_call_streaming(self, hy_v3_tool_parser, mock_request):
185
+ deltas = ["This is ", "a plain ", "response."]
186
+ results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
187
+ content = _collect_streaming_content(results)
188
+ assert content == "This is a plain response."
189
+ assert len(_collect_streaming_tool_calls(results)) == 0
190
+
191
+ def test_zero_arg_streaming(self, hy_v3_tool_parser, mock_request):
192
+ deltas = [
193
+ "<tool_calls>",
194
+ "\n<tool_call>",
195
+ "get_current_date",
196
+ "<tool_sep>",
197
+ "\n</tool_call>",
198
+ "\n</tool_calls>",
199
+ ]
200
+ results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
201
+ tc = _collect_streaming_tool_calls(results)
202
+ assert len(tc) == 1
203
+ assert tc[0]["name"] == "get_current_date"
204
+ assert json.loads(tc[0]["arguments"]) == {}
205
+
206
+ def test_args_streaming(self, hy_v3_tool_parser, mock_request):
207
+ deltas = [
208
+ "<tool_calls>",
209
+ "\n<tool_call>",
210
+ "get_weather",
211
+ "<tool_sep>",
212
+ "\n<arg_key>city</arg_key>",
213
+ "\n<arg_value>Beijing</arg_value>",
214
+ "\n<arg_key>date</arg_key>",
215
+ "\n<arg_value>2026-03-30</arg_value>",
216
+ "\n</tool_call>",
217
+ "\n</tool_calls>",
218
+ ]
219
+ results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
220
+ tc = _collect_streaming_tool_calls(results)
221
+ assert len(tc) == 1 and tc[0]["name"] == "get_weather"
222
+ assert json.loads(tc[0]["arguments"]) == {
223
+ "city": "Beijing",
224
+ "date": "2026-03-30",
225
+ }
226
+
227
+ def test_content_before_streaming(self, hy_v3_tool_parser, mock_request):
228
+ deltas = [
229
+ "Checking.",
230
+ "<tool_calls>",
231
+ "\n<tool_call>",
232
+ "get_current_date",
233
+ "<tool_sep>",
234
+ "\n</tool_call>",
235
+ "\n</tool_calls>",
236
+ ]
237
+ results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
238
+ assert "Checking." in _collect_streaming_content(results)
239
+ tc = _collect_streaming_tool_calls(results)
240
+ assert len(tc) == 1 and tc[0]["name"] == "get_current_date"
241
+
242
+ def test_multiple_streaming(self, hy_v3_tool_parser, mock_request):
243
+ deltas = [
244
+ "<tool_calls>",
245
+ "\n<tool_call>",
246
+ "get_weather",
247
+ "<tool_sep>",
248
+ "\n<arg_key>city</arg_key>",
249
+ "\n<arg_value>Beijing</arg_value>",
250
+ "\n<arg_key>date</arg_key>",
251
+ "\n<arg_value>2026-03-30</arg_value>",
252
+ "\n</tool_call>",
253
+ "\n<tool_call>",
254
+ "get_weather",
255
+ "<tool_sep>",
256
+ "\n<arg_key>city</arg_key>",
257
+ "\n<arg_value>Hangzhou</arg_value>",
258
+ "\n<arg_key>date</arg_key>",
259
+ "\n<arg_value>2026-03-30</arg_value>",
260
+ "\n</tool_call>",
261
+ "\n</tool_calls>",
262
+ ]
263
+ results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
264
+ tc = _collect_streaming_tool_calls(results)
265
+ assert len(tc) == 2
266
+ assert json.loads(tc[0]["arguments"])["city"] == "Beijing"
267
+ assert json.loads(tc[1]["arguments"])["city"] == "Hangzhou"
268
+
269
+ def test_all_in_one_delta_streaming(self, hy_v3_tool_parser, mock_request):
270
+ out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
271
+ results = _simulate_streaming(hy_v3_tool_parser, [out], mock_request)
272
+ tc = _collect_streaming_tool_calls(results)
273
+ assert len(tc) == 1 and tc[0]["name"] == "get_current_date"
274
+ assert json.loads(tc[0]["arguments"]) == {}
vllm/config/speculative.py
+8
@@ -47,6 +47,7 @@ MTPModelTypes = Literal[
47
"mtp",
48
"pangu_ultra_moe_mtp",
49
"step3p5_mtp",
50
+ "hy_v3_mtp",
51
]
52
NgramGPUTypes = Literal["ngram_gpu"]
53
DFlashModelTypes = Literal["dflash"]
@@ -364,6 +365,13 @@ class SpeculativeConfig:
365
if initial_architecture == "MistralLarge3ForCausalLM":
366
hf_config.update({"architectures": ["EagleMistralLarge3ForCausalLM"]})
367
368
+ if hf_config.model_type == "hy_v3":
369
+ hf_config.model_type = "hy_v3_mtp"
370
+ n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
371
+ hf_config.update(
372
+ {"n_predict": n_predict, "architectures": ["HYV3MTPModel"]}
373
+ )
374
+
375
return hf_config
376
377
def __post_init__(self):
vllm/model_executor/model_loader/weight_utils.py
+8
@@ -1562,6 +1562,11 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
1562
# NemotronH format: .mixer.{k,v}_proj.{k,v}_scale ->
1563
# .mixer.attn.{k,v}_scale
1564
(r"\.mixer\.[kv]_proj\.([kv])_scale$", r".mixer.attn.\1_scale"),
1565
+ # HYV3 format: .self_attn.q.scale -> .self_attn.attn.q_scale
1566
+ (r"\.self_attn\.q\.scale$", r".self_attn.attn.q_scale"),
1567
+ # HYV3 format: .self_attn.{k,v}_cache.scale ->
1568
+ # .self_attn.attn.{k,v}_scale
1569
+ (r"\.self_attn\.([kv])_cache\.scale$", r".self_attn.attn.\1_scale"),
1570
# Default format: .{k,v}_scale -> .attn.{k,v}_scale
1571
(r"\.([qkv])_scale$", r".attn.\1_scale"),
1572
(r"\.([qkv])_zero_point$", r".attn.\1_zero_point"),
@@ -1576,6 +1581,9 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
1581
".k_zero_point",
1582
".v_zero_point",
1583
".q_zero_point",
1584
+ ".q.scale",
1585
+ ".k_cache.scale",
1586
+ ".v_cache.scale",
1587
)
1588
):
1589
import regex as re
vllm/model_executor/models/hy_v3.py
new
+707
@@ -0,0 +1,707 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# coding=utf-8
5
+# Copyright 2026 The HY team.
6
+# Copyright 2023 The vLLM team.
7
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
8
+#
9
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
10
+# and OPT implementations in this library. It has been modified from its
11
+# original forms to accommodate minor architectural differences compared
12
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
13
+#
14
+# Licensed under the Apache License, Version 2.0 (the "License");
15
+# you may not use this file except in compliance with the License.
16
+# You may obtain a copy of the License at
17
+#
18
+# http://www.apache.org/licenses/LICENSE-2.0
19
+#
20
+# Unless required by applicable law or agreed to in writing, software
21
+# distributed under the License is distributed on an "AS IS" BASIS,
22
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23
+# See the License for the specific language governing permissions and
24
+# limitations under the License.
25
+"""Inference-only HY model compatible with HuggingFace weights."""
26
+
27
+import typing
28
+from collections.abc import Callable, Iterable
29
+from itertools import islice
30
+from typing import Any
31
+
32
+import torch
33
+from torch import nn
34
+from transformers import PretrainedConfig
35
+
36
+from vllm.compilation.decorators import support_torch_compile
37
+from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
38
+from vllm.distributed import (
39
+ get_ep_group,
40
+ get_pp_group,
41
+ get_tensor_model_parallel_world_size,
42
+)
43
+from vllm.logger import init_logger
44
+from vllm.model_executor.layers.activation import SiluAndMul
45
+from vllm.model_executor.layers.attention import Attention
46
+from vllm.model_executor.layers.fused_moe import FusedMoE, GateLinear
47
+from vllm.model_executor.layers.layernorm import RMSNorm
48
+from vllm.model_executor.layers.linear import (
49
+ MergedColumnParallelLinear,
50
+ QKVParallelLinear,
51
+ RowParallelLinear,
52
+)
53
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
54
+from vllm.model_executor.layers.quantization import QuantizationConfig
55
+from vllm.model_executor.layers.rotary_embedding import get_rope
56
+from vllm.model_executor.layers.vocab_parallel_embedding import (
57
+ ParallelLMHead,
58
+ VocabParallelEmbedding,
59
+)
60
+from vllm.model_executor.model_loader.weight_utils import (
61
+ default_weight_loader,
62
+ maybe_remap_kv_scale_name,
63
+)
64
+from vllm.sequence import IntermediateTensors
65
+from vllm.transformers_utils.configs.hy_v3 import HYV3Config
66
+
67
+from .interfaces import SupportsLoRA, SupportsPP
68
+from .utils import (
69
+ AutoWeightsLoader,
70
+ PPMissingLayer,
71
+ is_pp_missing_parameter,
72
+ make_empty_intermediate_tensors_factory,
73
+ make_layers,
74
+ maybe_prefix,
75
+)
76
+
77
+logger = init_logger(__name__)
78
+
79
+
80
+class HYV3FeedForward(nn.Module):
81
+ def __init__(
82
+ self,
83
+ hidden_size: int,
84
+ intermediate_size: int,
85
+ hidden_act: str,
86
+ quant_config: QuantizationConfig | None = None,
87
+ reduce_results: bool = True,
88
+ expert_gate: torch.nn.Linear | None = None,
89
+ prefix: str = "",
90
+ ) -> None:
91
+ super().__init__()
92
+ self.gate_up_proj = MergedColumnParallelLinear(
93
+ hidden_size,
94
+ [intermediate_size] * 2,
95
+ bias=False,
96
+ quant_config=quant_config,
97
+ prefix=f"{prefix}.gate_up_proj",
98
+ )
99
+ self.down_proj = RowParallelLinear(
100
+ intermediate_size,
101
+ hidden_size,
102
+ bias=False,
103
+ quant_config=quant_config,
104
+ reduce_results=reduce_results,
105
+ prefix=f"{prefix}.down_proj",
106
+ )
107
+ if hidden_act != "silu":
108
+ raise ValueError(
109
+ f"Unsupported activation: {hidden_act}. Only silu is supported for now."
110
+ )
111
+ self.act_fn = SiluAndMul()
112
+
113
+ def forward(self, x):
114
+ gate_up, _ = self.gate_up_proj(x)
115
+ out = self.act_fn(gate_up)
116
+ out, _ = self.down_proj(out)
117
+ return out
118
+
119
+
120
+class HYV3MoEFused(nn.Module):
121
+ def __init__(
122
+ self,
123
+ config: HYV3Config,
124
+ quant_config: QuantizationConfig | None = None,
125
+ prefix: str = "",
126
+ enable_eplb: bool = False,
127
+ ):
128
+ super().__init__()
129
+ self.tp_size = get_tensor_model_parallel_world_size()
130
+ self.ep_group = get_ep_group().device_group
131
+ self.ep_rank = get_ep_group().rank_in_group
132
+ self.ep_size = self.ep_group.size()
133
+ self.n_routed_experts = config.num_experts
134
+ if self.tp_size > config.num_experts:
135
+ raise ValueError(
136
+ f"Tensor parallel size {self.tp_size} is greater than "
137
+ f"the number of experts {config.num_experts}."
138
+ )
139
+ top_k = config.num_experts_per_tok
140
+ intermediate_size = config.expert_hidden_dim
141
+ router_scaling_factor = getattr(config, "router_scaling_factor", 1.0)
142
+ vllm_config = get_current_vllm_config()
143
+ eplb_config = vllm_config.parallel_config.eplb_config
144
+ self.enable_eplb = enable_eplb
145
+
146
+ self.n_logical_experts = self.n_routed_experts
147
+ self.n_redundant_experts = eplb_config.num_redundant_experts
148
+ self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
149
+ self.n_local_physical_experts = self.n_physical_experts // self.ep_size
150
+ self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
151
+ self.physical_expert_end = (
152
+ self.physical_expert_start + self.n_local_physical_experts
153
+ )
154
+ self.gate = GateLinear(
155
+ config.hidden_size,
156
+ config.num_experts,
157
+ bias=False,
158
+ out_dtype=torch.float32,
159
+ params_dtype=torch.float32,
160
+ prefix=f"{prefix}.gate",
161
+ )
162
+
163
+ if config.num_shared_experts > 0:
164
+ self.shared_mlp = HYV3FeedForward(
165
+ hidden_size=config.hidden_size,
166
+ intermediate_size=config.expert_hidden_dim * config.num_shared_experts,
167
+ hidden_act=config.hidden_act,
168
+ quant_config=quant_config,
169
+ prefix=f"{prefix}",
170
+ reduce_results=False,
171
+ )
172
+ else:
173
+ self.shared_mlp = None
174
+
175
+ self.expert_bias = nn.Parameter(torch.empty(config.num_experts))
176
+ scoring_func = "sigmoid"
177
+ e_score_correction_bias = self.expert_bias
178
+
179
+ self.experts = FusedMoE(
180
+ num_experts=self.n_routed_experts,
181
+ top_k=top_k,
182
+ hidden_size=config.hidden_size,
183
+ intermediate_size=intermediate_size,
184
+ renormalize=config.route_norm,
185
+ quant_config=quant_config,
186
+ prefix=f"{prefix}.experts",
187
+ enable_eplb=self.enable_eplb,
188
+ num_redundant_experts=self.n_redundant_experts,
189
+ scoring_func=scoring_func,
190
+ use_grouped_topk=True,
191
+ num_expert_group=1,
192
+ topk_group=1,
193
+ routed_scaling_factor=router_scaling_factor,
194
+ e_score_correction_bias=e_score_correction_bias,
195
+ n_shared_experts=config.num_shared_experts,
196
+ shared_experts=self.shared_mlp,
197
+ )
198
+
199
+ def forward(
200
+ self,
201
+ hidden_states: torch.Tensor,
202
+ ) -> torch.Tensor:
203
+ orig_shape = hidden_states.shape
204
+ hidden_dim = hidden_states.shape[-1]
205
+ hidden_states = hidden_states.view(-1, hidden_dim)
206
+
207
+ # router_logits: (num_tokens, n_experts)
208
+ router_logits, _ = self.gate(hidden_states)
209
+
210
+ final_hidden_states = self.experts(
211
+ hidden_states=hidden_states, router_logits=router_logits
212
+ )
213
+ return final_hidden_states.view(orig_shape)
214
+
215
+
216
+class HYV3Attention(nn.Module):
217
+ def __init__(
218
+ self,
219
+ config: PretrainedConfig,
220
+ hidden_size: int,
221
+ num_heads: int,
222
+ num_kv_heads: int,
223
+ rope_parameters: dict[str, Any],
224
+ max_position_embeddings: int = 8192,
225
+ head_dim: int | None = None,
226
+ rms_norm_eps: float = 1e-5,
227
+ cache_config: CacheConfig | None = None,
228
+ quant_config: QuantizationConfig | None = None,
229
+ prefix: str = "",
230
+ dual_chunk_attention_config: dict[str, Any] | None = None,
231
+ ) -> None:
232
+ super().__init__()
233
+ self.hidden_size = hidden_size
234
+ tp_size = get_tensor_model_parallel_world_size()
235
+ self.total_num_heads = num_heads
236
+ assert self.total_num_heads % tp_size == 0
237
+ self.num_heads = self.total_num_heads // tp_size
238
+ self.total_num_kv_heads = num_kv_heads
239
+ if self.total_num_kv_heads >= tp_size:
240
+ # Number of KV heads is greater than TP size, so we partition
241
+ # the KV heads across multiple tensor parallel GPUs.
242
+ assert self.total_num_kv_heads % tp_size == 0
243
+ else:
244
+ # Number of KV heads is less than TP size, so we replicate
245
+ # the KV heads across multiple tensor parallel GPUs.
246
+ assert tp_size % self.total_num_kv_heads == 0
247
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
248
+
249
+ if hasattr(config, "head_dim") and config.head_dim:
250
+ self.head_dim = config.head_dim
251
+ else:
252
+ self.head_dim = head_dim or (hidden_size // self.total_num_heads)
253
+ self.q_size = self.num_heads * self.head_dim
254
+ self.kv_size = self.num_kv_heads * self.head_dim
255
+ self.scaling = self.head_dim**-0.5
256
+ self.use_qk_norm = getattr(config, "qk_norm", False)
257
+ self.max_position_embeddings = max_position_embeddings
258
+
259
+ self.qkv_proj = QKVParallelLinear(
260
+ hidden_size,
261
+ self.head_dim,
262
+ self.total_num_heads,
263
+ self.total_num_kv_heads,
264
+ quant_config=quant_config,
265
+ bias=None,
266
+ prefix=f"{prefix}.qkv_proj",
267
+ )
268
+ self.o_proj = RowParallelLinear(
269
+ self.total_num_heads * self.head_dim,
270
+ hidden_size,
271
+ bias=False,
272
+ quant_config=quant_config,
273
+ prefix=f"{prefix}.o_proj",
274
+ )
275
+ self.rotary_emb = get_rope(
276
+ self.head_dim,
277
+ max_position=max_position_embeddings,
278
+ rope_parameters=rope_parameters,
279
+ is_neox_style=True,
280
+ )
281
+ self.attn = Attention(
282
+ self.num_heads,
283
+ self.head_dim,
284
+ self.scaling,
285
+ num_kv_heads=self.num_kv_heads,
286
+ cache_config=cache_config,
287
+ quant_config=quant_config,
288
+ prefix=f"{prefix}.attn",
289
+ )
290
+ if self.use_qk_norm:
291
+ self.q_norm = RMSNorm(self.head_dim, rms_norm_eps)
292
+ self.k_norm = RMSNorm(self.head_dim, rms_norm_eps)
293
+
294
+ def forward(
295
+ self,
296
+ positions: torch.Tensor,
297
+ hidden_states: torch.Tensor,
298
+ ) -> torch.Tensor:
299
+ qkv, _ = self.qkv_proj(hidden_states)
300
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
301
+ output_shape = None
302
+ if self.use_qk_norm:
303
+ q_by_head = q.view(
304
+ *q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim
305
+ )
306
+ q_by_head = self.q_norm(q_by_head)
307
+ q = q_by_head.view(q.shape)
308
+
309
+ k_by_head = k.view(
310
+ *k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim
311
+ )
312
+ k_by_head = self.k_norm(k_by_head)
313
+ k = k_by_head.view(k.shape)
314
+ q, k = self.rotary_emb(positions, q, k)
315
+ attn_output = self.attn(q, k, v, output_shape)
316
+ attn_output = attn_output.view(q.shape[0], -1)
317
+ output, _ = self.o_proj(attn_output)
318
+ return output
319
+
320
+
321
+class HYV3DecoderLayer(nn.Module):
322
+ def __init__(
323
+ self,
324
+ config: PretrainedConfig,
325
+ cache_config: CacheConfig | None = None,
326
+ quant_config: QuantizationConfig | None = None,
327
+ prefix: str = "",
328
+ ) -> None:
329
+ super().__init__()
330
+ self.hidden_size = config.hidden_size
331
+ layer_idx = int(prefix.split(".")[-1])
332
+ max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
333
+ self.self_attn = HYV3Attention(
334
+ config=config,
335
+ hidden_size=self.hidden_size,
336
+ num_heads=config.num_attention_heads,
337
+ num_kv_heads=config.num_key_value_heads,
338
+ rope_parameters=config.rope_parameters,
339
+ max_position_embeddings=max_position_embeddings,
340
+ head_dim=config.head_dim,
341
+ rms_norm_eps=config.rms_norm_eps,
342
+ cache_config=cache_config,
343
+ quant_config=quant_config,
344
+ prefix=f"{prefix}.self_attn",
345
+ )
346
+ self.input_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
347
+ self.post_attention_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
348
+ if not hasattr(config, "first_k_dense_replace"):
349
+ raise ValueError("first_k_dense_replace not exist,please check config")
350
+ if layer_idx < config.first_k_dense_replace:
351
+ self.mlp = HYV3FeedForward(
352
+ hidden_size=config.hidden_size,
353
+ intermediate_size=config.intermediate_size,
354
+ hidden_act=config.hidden_act,
355
+ quant_config=quant_config,
356
+ prefix=f"{prefix}.mlp",
357
+ )
358
+ self.block_type = "feedforward"
359
+ else:
360
+ self.mlp = HYV3MoEFused(
361
+ config=config, quant_config=quant_config, prefix=f"{prefix}.mlp"
362
+ )
363
+ self.block_type = "moe"
364
+
365
+ def forward(
366
+ self,
367
+ positions: torch.Tensor,
368
+ hidden_states: torch.Tensor,
369
+ residual: torch.Tensor | None,
370
+ idx: int = -1,
371
+ ) -> torch.Tensor:
372
+ if residual is None:
373
+ residual = hidden_states
374
+ hidden_states = self.input_layernorm(hidden_states)
375
+ else:
376
+ hidden_states, residual = self.input_layernorm(hidden_states, residual)
377
+
378
+ hidden_states = self.self_attn(
379
+ positions=positions,
380
+ hidden_states=hidden_states,
381
+ )
382
+
383
+ hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
384
+
385
+ hidden_states = self.mlp(hidden_states)
386
+
387
+ return hidden_states, residual
388
+
389
+
390
+@support_torch_compile
391
+class HYV3Model(nn.Module):
392
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
393
+ super().__init__()
394
+
395
+ config = vllm_config.model_config.hf_config
396
+ cache_config = vllm_config.cache_config
397
+ quant_config = vllm_config.quant_config
398
+
399
+ parallel_config = vllm_config.parallel_config
400
+ eplb_config = parallel_config.eplb_config
401
+ self.num_redundant_experts = eplb_config.num_redundant_experts
402
+
403
+ self.vocab_size = config.vocab_size
404
+ self.config = config
405
+ self.quant_config = quant_config
406
+
407
+ self.embed_tokens = VocabParallelEmbedding(
408
+ config.vocab_size,
409
+ config.hidden_size,
410
+ )
411
+
412
+ self.start_layer, self.end_layer, self.layers = make_layers(
413
+ config.num_hidden_layers,
414
+ lambda prefix: HYV3DecoderLayer(
415
+ config=config,
416
+ cache_config=cache_config,
417
+ quant_config=quant_config,
418
+ prefix=prefix,
419
+ ),
420
+ prefix=f"{prefix}.layers",
421
+ )
422
+ self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
423
+ self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
424
+ ["hidden_states", "residual"], config.hidden_size
425
+ )
426
+
427
+ # Set MoE hyperparameters
428
+ self.expert_weights = []
429
+ self.num_expert_groups = 1
430
+ self.moe_layers = []
431
+ example_layer = None
432
+ for layer in self.layers:
433
+ if isinstance(layer, PPMissingLayer):
434
+ continue
435
+
436
+ assert isinstance(layer, HYV3DecoderLayer)
437
+ if layer.block_type == "moe":
438
+ example_layer = layer.mlp
439
+ self.moe_layers.append(layer.mlp.experts)
440
+
441
+ if example_layer is None:
442
+ self.num_moe_layers = 0
443
+ raise RuntimeError("No MoE layer found in model.layers.")
444
+
445
+ self.num_moe_layers = len(self.moe_layers)
446
+ self.num_logical_experts = getattr(example_layer, "n_logical_experts", None)
447
+ self.num_physical_experts = getattr(example_layer, "n_physical_experts", None)
448
+ self.num_local_physical_experts = getattr(
449
+ example_layer, "n_local_physical_experts", None
450
+ )
451
+ self.num_routed_experts = getattr(example_layer, "n_routed_experts", None)
452
+ self.num_redundant_experts = getattr(example_layer, "n_redundant_experts", None)
453
+
454
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
455
+ return self.embed_tokens(input_ids)
456
+
457
+ def update_physical_experts_metadata(
458
+ self,
459
+ num_physical_experts: int,
460
+ num_local_physical_experts: int,
461
+ ) -> None:
462
+ assert self.num_local_physical_experts == num_local_physical_experts
463
+ self.num_physical_experts = num_physical_experts
464
+ self.num_local_physical_experts = num_local_physical_experts
465
+ self.num_redundant_experts = num_physical_experts - self.num_logical_experts
466
+ for layer in self.layers:
467
+ if isinstance(layer.mlp, HYV3MoEFused):
468
+ moe = layer.mlp
469
+ moe.n_local_physical_experts = num_local_physical_experts
470
+ moe.n_physical_experts = num_physical_experts
471
+ moe.n_redundant_experts = self.num_redundant_experts
472
+ moe.experts.update_expert_map()
473
+
474
+ def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
475
+ # Params for weights, fp8 weight scales, fp8 activation scales
476
+ # (param_name, weight_name, expert_id, shard_id)
477
+ return FusedMoE.make_expert_params_mapping(
478
+ self,
479
+ ckpt_gate_proj_name="gate_proj",
480
+ ckpt_down_proj_name="down_proj",
481
+ ckpt_up_proj_name="up_proj",
482
+ num_experts=self.config.num_experts,
483
+ )
484
+
485
+ def forward(
486
+ self,
487
+ input_ids: torch.Tensor,
488
+ positions: torch.Tensor,
489
+ intermediate_tensors: IntermediateTensors | None = None,
490
+ inputs_embeds: torch.Tensor | None = None,
491
+ ) -> torch.Tensor | IntermediateTensors:
492
+ if get_pp_group().is_first_rank:
493
+ if inputs_embeds is not None:
494
+ hidden_states = inputs_embeds
495
+ else:
496
+ hidden_states = self.embed_input_ids(input_ids)
497
+ residual = None
498
+ else:
499
+ assert intermediate_tensors is not None
500
+ hidden_states = intermediate_tensors["hidden_states"]
501
+ residual = intermediate_tensors["residual"]
502
+
503
+ for idx, layer in enumerate(
504
+ islice(self.layers, self.start_layer, self.end_layer)
505
+ ):
506
+ hidden_states, residual = layer(positions, hidden_states, residual, idx=idx)
507
+ if not get_pp_group().is_last_rank:
508
+ return IntermediateTensors(
509
+ {"hidden_states": hidden_states, "residual": residual}
510
+ )
511
+
512
+ hidden_states = hidden_states + residual
513
+ residual = hidden_states
514
+
515
+ hidden_states = self.norm(hidden_states)
516
+
517
+ return hidden_states
518
+
519
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
520
+ stacked_params_mapping = [
521
+ # (param_name, shard_name, shard_id)
522
+ (".qkv_proj", ".q_proj", "q"),
523
+ (".qkv_proj", ".k_proj", "k"),
524
+ (".qkv_proj", ".v_proj", "v"),
525
+ (".gate_up_proj", ".gate_proj", 0),
526
+ (".gate_up_proj", ".up_proj", 1),
527
+ ]
528
+ params_dict = dict(self.named_parameters())
529
+ expert_params_mapping = self.get_expert_mapping()
530
+ loaded_params: set[str] = set()
531
+ for name, loaded_weight in weights:
532
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
533
+ continue
534
+ if self.quant_config is not None and (
535
+ scale_name := self.quant_config.get_cache_scale(name)
536
+ ):
537
+ param = params_dict[scale_name]
538
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
539
+ loaded_weight = (
540
+ loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
541
+ )
542
+ weight_loader(param, loaded_weight)
543
+ loaded_params.add(scale_name)
544
+ continue
545
+ if "scale" in name:
546
+ # Remapping the name of FP8 kv-scale.
547
+ name = maybe_remap_kv_scale_name(name, params_dict)
548
+ if name is None:
549
+ continue
550
+ is_found = False
551
+ for param_name, weight_name, shard_id in stacked_params_mapping:
552
+ if weight_name not in name:
553
+ continue
554
+ if "mlp.experts" in name:
555
+ continue
556
+ name = name.replace(weight_name, param_name)
557
+ # Skip loading extra bias for GPTQ models.
558
+ if name.endswith(".bias") and name not in params_dict:
559
+ continue
560
+
561
+ # Skip layers on other devices.
562
+ if is_pp_missing_parameter(name, self):
563
+ continue
564
+
565
+ param = params_dict[name]
566
+ weight_loader = param.weight_loader
567
+ weight_loader(param, loaded_weight, shard_id)
568
+ loaded_params.add(name)
569
+ is_found = True
570
+ break
571
+ if is_found:
572
+ continue
573
+
574
+ if name.endswith(".bias") and name not in params_dict:
575
+ continue
576
+ is_expert_weight = False
577
+ for mapping in expert_params_mapping:
578
+ param_name, weight_name, expert_id, shard_id = mapping
579
+ if weight_name not in name:
580
+ continue
581
+ is_expert_weight = True
582
+ name_mapped = name.replace(weight_name, param_name)
583
+ # Skip layers on other devices.
584
+ if is_pp_missing_parameter(name_mapped, self):
585
+ continue
586
+
587
+ param = params_dict[name_mapped]
588
+ weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
589
+ success = weight_loader(
590
+ param,
591
+ loaded_weight,
592
+ name_mapped,
593
+ shard_id=shard_id,
594
+ expert_id=expert_id,
595
+ return_success=True,
596
+ )
597
+ if success:
598
+ name = name_mapped
599
+ break
600
+ else:
601
+ if is_expert_weight:
602
+ # We've checked that this is an expert weight
603
+ # However it's not mapped locally to this rank
604
+ # So we simply skip it
605
+ continue
606
+ if name is None:
607
+ continue
608
+ if is_pp_missing_parameter(name, self):
609
+ continue
610
+ if "router.gate." in name:
611
+ name = name.replace("router.", "")
612
+
613
+ param = params_dict[name]
614
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
615
+ weight_loader(param, loaded_weight)
616
+ loaded_params.add(name)
617
+
618
+ return loaded_params
619
+
620
+
621
+def get_spec_layer_idx_from_weight_name(
622
+ config: PretrainedConfig, weight_name: str
623
+) -> int | None:
624
+ # HYV3MTP is enabled only when num_nextn_predict_layers is greater than 1
625
+ if (
626
+ hasattr(config, "num_nextn_predict_layers")
627
+ and config.num_nextn_predict_layers > 0
628
+ ):
629
+ layer_idx = config.num_hidden_layers
630
+ for i in range(config.num_nextn_predict_layers):
631
+ if weight_name.startswith(f"model.layers.{layer_idx + i}."):
632
+ return layer_idx + i
633
+ return None
634
+
635
+
636
+class HYV3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
637
+ packed_modules_mapping = {
638
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
639
+ "gate_up_proj": ["gate_proj", "up_proj"],
640
+ }
641
+
642
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
643
+ super().__init__()
644
+ config = vllm_config.model_config.hf_config
645
+ quant_config = vllm_config.quant_config
646
+ self.config = config
647
+ self.quant_config = quant_config
648
+
649
+ parallel_config = vllm_config.parallel_config
650
+ eplb_config = parallel_config.eplb_config
651
+ self.num_redundant_experts = eplb_config.num_redundant_experts
652
+
653
+ self.model = HYV3Model(
654
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
655
+ )
656
+ self.lm_head = ParallelLMHead(
657
+ config.vocab_size,
658
+ config.hidden_size,
659
+ quant_config=quant_config,
660
+ prefix=maybe_prefix(prefix, "lm_head"),
661
+ )
662
+ if self.config.tie_word_embeddings:
663
+ self.lm_head.weight = self.model.embed_tokens.weight
664
+ self.logits_processor = LogitsProcessor(config.vocab_size)
665
+ self.make_empty_intermediate_tensors = (
666
+ self.model.make_empty_intermediate_tensors
667
+ )
668
+
669
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
670
+ return self.model.embed_input_ids(input_ids)
671
+
672
+ def forward(
673
+ self,
674
+ input_ids: torch.Tensor,
675
+ positions: torch.Tensor,
676
+ intermediate_tensors: IntermediateTensors | None = None,
677
+ inputs_embeds: torch.Tensor | None = None,
678
+ ) -> torch.Tensor | IntermediateTensors:
679
+ hidden_states = self.model(
680
+ input_ids, positions, intermediate_tensors, inputs_embeds
681
+ )
682
+
683
+ return hidden_states
684
+
685
+ def compute_logits(
686
+ self,
687
+ hidden_states: torch.Tensor,
688
+ ) -> torch.Tensor | None:
689
+ logits = self.logits_processor(self.lm_head, hidden_states)
690
+ return logits
691
+
692
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
693
+ def _filter_weights(weights):
694
+ for name, weight in weights:
695
+ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
696
+ if spec_layer is not None:
697
+ continue
698
+ yield name, weight
699
+
700
+ loader = AutoWeightsLoader(
701
+ self,
702
+ skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
703
+ )
704
+ return loader.load_weights(_filter_weights(weights))
705
+
706
+ def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
707
+ return self.model.get_expert_mapping()
vllm/model_executor/models/hy_v3_mtp.py
new
+470
@@ -0,0 +1,470 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# coding=utf-8
5
+# Copyright 2026 The HY team.
6
+# Copyright 2023 The vLLM team.
7
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
8
+#
9
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
10
+# and OPT implementations in this library. It has been modified from its
11
+# original forms to accommodate minor architectural differences compared
12
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
13
+#
14
+# Licensed under the Apache License, Version 2.0 (the "License");
15
+# you may not use this file except in compliance with the License.
16
+# You may obtain a copy of the License at
17
+#
18
+# http://www.apache.org/licenses/LICENSE-2.0
19
+#
20
+# Unless required by applicable law or agreed to in writing, software
21
+# distributed under the License is distributed on an "AS IS" BASIS,
22
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23
+# See the License for the specific language governing permissions and
24
+# limitations under the License.
25
+"""Inference-only HY V3 MTP model compatible with HuggingFace weights."""
26
+
27
+from collections.abc import Iterable
28
+
29
+import regex as re
30
+import torch
31
+from torch import nn
32
+from transformers import PretrainedConfig
33
+
34
+from vllm.config import CacheConfig, ModelConfig, VllmConfig
35
+from vllm.model_executor.layers.fused_moe import FusedMoE
36
+from vllm.model_executor.layers.layernorm import RMSNorm
37
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.quantization import QuantizationConfig
39
+from vllm.model_executor.layers.vocab_parallel_embedding import (
40
+ ParallelLMHead,
41
+ VocabParallelEmbedding,
42
+)
43
+from vllm.model_executor.model_loader.weight_utils import (
44
+ default_weight_loader,
45
+ maybe_remap_kv_scale_name,
46
+)
47
+from vllm.sequence import IntermediateTensors
48
+from vllm.v1.outputs import SamplerOutput
49
+from vllm.v1.sample.metadata import SamplingMetadata
50
+from vllm.v1.sample.sampler import Sampler
51
+
52
+from .hy_v3 import HYV3DecoderLayer, get_spec_layer_idx_from_weight_name
53
+from .utils import is_pp_missing_parameter, maybe_prefix
54
+
55
+
56
+def _is_moe(config: PretrainedConfig) -> bool:
57
+ return bool(
58
+ getattr(config, "num_experts", None)
59
+ and (
60
+ (isinstance(config.num_experts, int) and config.num_experts > 1)
61
+ or (isinstance(config.num_experts, list) and max(config.num_experts) > 1)
62
+ )
63
+ )
64
+
65
+
66
+def _get_cla_factor(config: PretrainedConfig) -> int:
67
+ if not getattr(config, "use_cla", False):
68
+ return 1
69
+ return getattr(config, "cla_share_factor", 1)
70
+
71
+
72
+class HYV3SharedHead(nn.Module):
73
+ def __init__(
74
+ self,
75
+ config: PretrainedConfig,
76
+ quant_config: QuantizationConfig | None = None,
77
+ ) -> None:
78
+ super().__init__()
79
+ self.head = ParallelLMHead(
80
+ config.vocab_size, config.hidden_size, quant_config=quant_config
81
+ )
82
+
83
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
84
+ return hidden_states
85
+
86
+
87
+class HYV3MultiTokenPredictorLayer(nn.Module):
88
+ def __init__(
89
+ self,
90
+ config: PretrainedConfig,
91
+ prefix: str,
92
+ model_config: ModelConfig,
93
+ cache_config: CacheConfig | None = None,
94
+ quant_config: QuantizationConfig | None = None,
95
+ ) -> None:
96
+ super().__init__()
97
+
98
+ self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
99
+ self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
100
+ self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
101
+ self.shared_head = HYV3SharedHead(config=config, quant_config=quant_config)
102
+ self.mtp_block = HYV3DecoderLayer(
103
+ config=config,
104
+ cache_config=cache_config,
105
+ quant_config=quant_config,
106
+ prefix=prefix,
107
+ )
108
+ # Final layernorm applied after transformer block, before logits
109
+ # projection (matches HF HYV3MTPDecoderLayer.final_layernorm)
110
+ self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
111
+
112
+ def forward(
113
+ self,
114
+ input_ids: torch.Tensor,
115
+ positions: torch.Tensor,
116
+ previous_hidden_states: torch.Tensor,
117
+ inputs_embeds: torch.Tensor | None = None,
118
+ spec_step_index: int = 0,
119
+ ) -> torch.Tensor:
120
+ assert inputs_embeds is not None
121
+ # masking inputs at position 0, as not needed by MTP
122
+ inputs_embeds[positions == 0] = 0
123
+ inputs_embeds = self.enorm(inputs_embeds)
124
+ previous_hidden_states = self.hnorm(previous_hidden_states)
125
+
126
+ hidden_states = self.eh_proj(
127
+ torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
128
+ )
129
+
130
+ # HYV3DecoderLayer returns (hidden_states, residual)
131
+ hidden_states, residual = self.mtp_block(
132
+ positions=positions, hidden_states=hidden_states, residual=None
133
+ )
134
+ hidden_states = residual + hidden_states
135
+ hidden_states = self.final_layernorm(hidden_states)
136
+ return hidden_states
137
+
138
+
139
+class HYV3MultiTokenPredictor(nn.Module):
140
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
141
+ super().__init__()
142
+ config = vllm_config.model_config.hf_config
143
+ self.mtp_start_layer_idx = config.num_hidden_layers
144
+ self.num_mtp_layers = config.num_nextn_predict_layers
145
+
146
+ # to map the exact layer index from weights
147
+ self.layers = torch.nn.ModuleDict(
148
+ {
149
+ str(idx): HYV3MultiTokenPredictorLayer(
150
+ config,
151
+ f"{prefix}.layers.{idx}",
152
+ model_config=vllm_config.model_config,
153
+ cache_config=vllm_config.cache_config,
154
+ quant_config=vllm_config.quant_config,
155
+ )
156
+ for idx in range(
157
+ self.mtp_start_layer_idx,
158
+ self.mtp_start_layer_idx + self.num_mtp_layers,
159
+ )
160
+ }
161
+ )
162
+
163
+ self.embed_tokens = VocabParallelEmbedding(
164
+ config.vocab_size,
165
+ config.hidden_size,
166
+ )
167
+
168
+ self.logits_processor = LogitsProcessor(config.vocab_size)
169
+
170
+ def forward(
171
+ self,
172
+ input_ids: torch.Tensor,
173
+ positions: torch.Tensor,
174
+ previous_hidden_states: torch.Tensor,
175
+ inputs_embeds: torch.Tensor | None = None,
176
+ spec_step_idx: int = 0,
177
+ ) -> torch.Tensor:
178
+ if inputs_embeds is None:
179
+ inputs_embeds = self.embed_tokens(input_ids)
180
+ current_step_idx = spec_step_idx % self.num_mtp_layers
181
+ return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
182
+ input_ids,
183
+ positions,
184
+ previous_hidden_states,
185
+ inputs_embeds,
186
+ current_step_idx,
187
+ )
188
+
189
+ def compute_logits(
190
+ self,
191
+ hidden_states: torch.Tensor,
192
+ spec_step_idx: int = 0,
193
+ ) -> torch.Tensor:
194
+ current_step_idx = spec_step_idx % self.num_mtp_layers
195
+ mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
196
+ logits = self.logits_processor(
197
+ mtp_layer.shared_head.head, mtp_layer.shared_head(hidden_states)
198
+ )
199
+ return logits
200
+
201
+
202
+class HYV3MTP(nn.Module):
203
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
204
+ super().__init__()
205
+ self.config = vllm_config.model_config.hf_config
206
+ self.quant_config = vllm_config.quant_config
207
+ self.model = HYV3MultiTokenPredictor(
208
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
209
+ )
210
+
211
+ self.sampler = Sampler()
212
+
213
+ def forward(
214
+ self,
215
+ input_ids: torch.Tensor,
216
+ positions: torch.Tensor,
217
+ hidden_states: torch.Tensor,
218
+ intermediate_tensors: IntermediateTensors | None = None,
219
+ inputs_embeds: torch.Tensor | None = None,
220
+ spec_step_idx: int = 0,
221
+ ) -> torch.Tensor:
222
+ hidden_states = self.model(
223
+ input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
224
+ )
225
+ return hidden_states
226
+
227
+ def compute_logits(
228
+ self,
229
+ hidden_states: torch.Tensor,
230
+ spec_step_idx: int = 0,
231
+ ) -> torch.Tensor | None:
232
+ return self.model.compute_logits(hidden_states, spec_step_idx)
233
+
234
+ def sample(
235
+ self,
236
+ logits: torch.Tensor,
237
+ sampling_metadata: SamplingMetadata,
238
+ ) -> SamplerOutput | None:
239
+ next_tokens = self.sampler(logits, sampling_metadata)
240
+ return next_tokens
241
+
242
+ def _split_qkv_weight(self, qkv: torch.Tensor):
243
+ num_attention_heads = self.config.num_attention_heads
244
+ num_kv_heads = getattr(
245
+ self.config, "num_key_value_heads", self.config.num_attention_heads
246
+ )
247
+ num_key_value_groups = num_attention_heads // num_kv_heads
248
+ hidden_size = self.config.hidden_size
249
+
250
+ if hasattr(self.config, "head_dim"):
251
+ attention_head_dim = self.config.head_dim
252
+ elif hasattr(self.config, "attention_head_dim"):
253
+ attention_head_dim = self.config.attention_head_dim
254
+ else:
255
+ attention_head_dim = self.config.hidden_size // num_attention_heads
256
+
257
+ qkv = qkv.reshape(
258
+ num_kv_heads, num_key_value_groups + 2, attention_head_dim, hidden_size
259
+ )
260
+ q, k, v = torch.split(qkv, (num_key_value_groups, 1, 1), dim=1)
261
+ q = q.reshape(-1, hidden_size)
262
+ k = k.reshape(-1, hidden_size)
263
+ v = v.reshape(-1, hidden_size)
264
+ return torch.concat((q, k, v))
265
+
266
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
267
+ cla_factor = _get_cla_factor(self.config)
268
+ stacked_params_mapping = [
269
+ # (param_name, shard_name, shard_id)
270
+ (".qkv_proj", ".q_proj", "q"),
271
+ (".qkv_proj", ".k_proj", "k"),
272
+ (".qkv_proj", ".v_proj", "v"),
273
+ (".gate_up_proj", ".gate_proj", 0),
274
+ (".gate_up_proj", ".up_proj", 1),
275
+ ]
276
+
277
+ num_attention_heads = self.config.num_attention_heads
278
+ num_kv_heads = getattr(
279
+ self.config, "num_key_value_heads", self.config.num_attention_heads
280
+ )
281
+ split_params_mapping = [
282
+ (".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None),
283
+ (
284
+ ".qkv_proj",
285
+ ".qkv_proj",
286
+ num_attention_heads + num_kv_heads * 2,
287
+ [("q", num_attention_heads), ("k", num_kv_heads), ("v", num_kv_heads)],
288
+ self._split_qkv_weight,
289
+ ),
290
+ ]
291
+
292
+ if _is_moe(self.config):
293
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
294
+ self,
295
+ ckpt_gate_proj_name="gate_proj",
296
+ ckpt_down_proj_name="down_proj",
297
+ ckpt_up_proj_name="up_proj",
298
+ num_experts=self.config.num_experts,
299
+ )
300
+ else:
301
+ expert_params_mapping = {}
302
+
303
+ params_dict = dict(self.named_parameters())
304
+
305
+ # V3 shared weights mapping:
306
+ # - embed_tokens: from main model's model.embed_tokens.weight
307
+ # - lm_head: from main model's lm_head.weight → MTP shared_head.head
308
+ # (HF infer_mtp uses head_weight=self.lm_head.weight, not the
309
+ # checkpoint's model.layers.<N>.shared_head.weight)
310
+ # - No norm mapping (V3 MTP has no intermediate norm before lm_head)
311
+ mtp_start = self.config.num_hidden_layers
312
+ v3_shared_weights = {
313
+ "model.embed_tokens.weight": "model.embed_tokens.weight",
314
+ "lm_head.weight": f"model.layers.{mtp_start}.shared_head.head.weight",
315
+ }
316
+
317
+ for name, loaded_weight in weights:
318
+ # Intercept shared weights before any other processing
319
+ if name in v3_shared_weights:
320
+ target_name = v3_shared_weights[name]
321
+ if target_name in params_dict:
322
+ param = params_dict[target_name]
323
+ weight_loader = getattr(
324
+ param, "weight_loader", default_weight_loader
325
+ )
326
+ weight_loader(param, loaded_weight)
327
+ continue
328
+
329
+ if "rotary_emb.inv_freq" in name:
330
+ continue
331
+ if "gate_proj_bias" in name:
332
+ name = name.replace("gate_proj_bias", "gate_proj.bias")
333
+ if "up_proj_bias" in name:
334
+ name = name.replace("up_proj_bias", "up_proj.bias")
335
+ if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
336
+ continue
337
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
338
+ continue
339
+ if self.quant_config is not None and (
340
+ scale_name := self.quant_config.get_cache_scale(name)
341
+ ):
342
+ param = params_dict[scale_name]
343
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
344
+ loaded_weight = loaded_weight[0]
345
+ weight_loader(param, loaded_weight)
346
+ continue
347
+ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
348
+ if spec_layer is None:
349
+ continue
350
+ name = self._rewrite_spec_layer_name(spec_layer, name)
351
+ # Skip weights that _rewrite_spec_layer_name marked for skipping
352
+ if name == "__skip__":
353
+ continue
354
+ if "scale" in name:
355
+ name = maybe_remap_kv_scale_name(name, params_dict)
356
+ if name is None:
357
+ continue
358
+ is_found = False
359
+
360
+ for param_name, weight_name, shard_id in stacked_params_mapping:
361
+ if weight_name not in name:
362
+ continue
363
+ if "mlp.experts" in name:
364
+ continue
365
+ if weight_name == ".q_proj":
366
+ match = re.search(r"layers\.\d+", name)
367
+ if match:
368
+ layer_id = int(match.group(0).split(".")[-1])
369
+ if cla_factor > 1 and layer_id % cla_factor != 0:
370
+ continue
371
+ name = name.replace(weight_name, param_name)
372
+ if name.endswith(".bias") and name not in params_dict:
373
+ continue
374
+
375
+ if is_pp_missing_parameter(name, self):
376
+ continue
377
+
378
+ param = params_dict[name]
379
+ weight_loader = param.weight_loader
380
+ weight_loader(param, loaded_weight, shard_id)
381
+
382
+ is_found = True
383
+ break
384
+ if is_found:
385
+ continue
386
+
387
+ for param_name, weight_name, den, split_param, func in split_params_mapping:
388
+ if weight_name not in name:
389
+ continue
390
+ name = name.replace(weight_name, param_name)
391
+ if name.endswith(".bias") and name not in params_dict:
392
+ continue
393
+
394
+ if is_pp_missing_parameter(name, self):
395
+ continue
396
+
397
+ assert loaded_weight.shape[0] % den == 0
398
+ units = loaded_weight.shape[0] // den
399
+
400
+ param = params_dict[name]
401
+ weight_loader = param.weight_loader
402
+ offset = 0
403
+ for shard_id, num in split_param:
404
+ new_offset = offset + num * units
405
+ if func:
406
+ weight_loader(
407
+ param, func(loaded_weight)[offset:new_offset], shard_id
408
+ )
409
+ else:
410
+ weight_loader(param, loaded_weight[offset:new_offset], shard_id)
411
+ offset = new_offset
412
+
413
+ break
414
+ else:
415
+ if name.endswith(".bias") and name not in params_dict:
416
+ continue
417
+ for mapping in expert_params_mapping:
418
+ param_name, weight_name, expert_id, shard_id = mapping
419
+ if weight_name not in name:
420
+ continue
421
+ name = name.replace(weight_name, param_name)
422
+ if is_pp_missing_parameter(name, self):
423
+ continue
424
+ param = params_dict[name]
425
+ weight_loader = param.weight_loader
426
+ weight_loader(
427
+ param,
428
+ loaded_weight,
429
+ name,
430
+ shard_id=shard_id,
431
+ expert_id=expert_id,
432
+ )
433
+ break
434
+ else:
435
+ if is_pp_missing_parameter(name, self):
436
+ continue
437
+
438
+ if "mlp.gate.wg." in name:
439
+ name = name.replace("wg.", "")
440
+ # V3 checkpoint: mlp.router.gate -> mlp.gate
441
+ if "mlp.router.gate." in name:
442
+ name = name.replace("router.gate.", "gate.")
443
+
444
+ param = params_dict[name]
445
+ weight_loader = getattr(
446
+ param, "weight_loader", default_weight_loader
447
+ )
448
+ weight_loader(param, loaded_weight)
449
+
450
+ def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
451
+ """Rewrite spec layer weight names to match vLLM module structure."""
452
+ # Skip embed_tokens (doesn't exist in V3 MTP checkpoint under spec
453
+ # layer) and shared_head (we use main model's lm_head instead)
454
+ if f"model.layers.{spec_layer}.embed_tokens" in name:
455
+ return "__skip__"
456
+ if f"model.layers.{spec_layer}.shared_head" in name:
457
+ return "__skip__"
458
+
459
+ spec_layer_weight_names = ["enorm", "hnorm", "eh_proj", "final_layernorm"]
460
+ spec_layer_weight = False
461
+ for weight_name in spec_layer_weight_names:
462
+ if weight_name in name:
463
+ spec_layer_weight = True
464
+ break
465
+ if not spec_layer_weight:
466
+ # Transformer block weights go under .mtp_block
467
+ name = name.replace(
468
+ f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
469
+ )
470
+ return name
vllm/model_executor/models/registry.py
+2
@@ -133,6 +133,7 @@ _TEXT_GENERATION_MODELS = {
133
"Grok1ForCausalLM": ("grok1", "GrokForCausalLM"),
134
"HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"),
135
"HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
136
+ "HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
137
"HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
138
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
139
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
@@ -599,6 +600,7 @@ _SPECULATIVE_DECODING_MODELS = {
600
"Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
601
"Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
602
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
603
+ "HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
604
# Temporarily disabled.
605
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
606
# "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
vllm/reasoning/__init__.py
+4
@@ -56,6 +56,10 @@ _REASONING_PARSERS_TO_REGISTER = {
56
"hunyuan_a13b_reasoning_parser",
57
"HunyuanA13BReasoningParser",
58
),
59
+ "hy_v3": (
60
+ "hy_v3_reasoning_parser",
61
+ "HYV3ReasoningParser",
62
+ ),
63
"kimi_k2": (
64
"kimi_k2_reasoning_parser",
65
"KimiK2ReasoningParser",
vllm/reasoning/hy_v3_reasoning_parser.py
new
+137
@@ -0,0 +1,137 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Iterable, Sequence
5
+
6
+from vllm.entrypoints.openai.chat_completion.protocol import (
7
+ ChatCompletionRequest,
8
+)
9
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
10
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
11
+from vllm.logger import init_logger
12
+from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
13
+from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
14
+from vllm.tokenizers import TokenizerLike
15
+
16
+logger = init_logger(__name__)
17
+
18
+
19
+class HYV3ReasoningParser(BaseThinkingReasoningParser):
20
+ """
21
+ HYV3 parser that delegates to either HYV3ReasoningParser or
22
+ IdentityReasoningParser based on `reasoning_effort`.
23
+
24
+ The HYV3 model uses <think>...</think> tokens to denote reasoning text.
25
+ This parser extracts the reasoning content from the model output.
26
+ """
27
+
28
+ def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
29
+ super().__init__(tokenizer, *args, **kwargs)
30
+
31
+ # First, If there is reasoning_effort in chat_kwargs,
32
+ # prioritize using chat_kwargs.reasoning_effort.
33
+ # If it's not present, use the "reasoning_effort" field
34
+ # at the outer level of the chat message.
35
+ # Otherwise, If both are empty, assign "no_think".
36
+
37
+ chat_kwargs = kwargs.pop("chat_template_kwargs", {}) or {}
38
+ reasoning_effort = chat_kwargs.pop("reasoning_effort", "no_think")
39
+
40
+ logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort)
41
+
42
+ self._identity_parser: IdentityReasoningParser | None
43
+ if reasoning_effort == "no_think":
44
+ self._identity_parser = IdentityReasoningParser(tokenizer, *args, **kwargs)
45
+ else:
46
+ self._identity_parser = None
47
+
48
+ @property
49
+ def start_token(self) -> str:
50
+ """The token that starts reasoning content."""
51
+ return "<think>"
52
+
53
+ @property
54
+ def end_token(self) -> str:
55
+ """The token that ends reasoning content."""
56
+ return "</think>"
57
+
58
+ def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
59
+ if self._identity_parser is not None:
60
+ return self._identity_parser.is_reasoning_end(input_ids)
61
+
62
+ return super().is_reasoning_end(input_ids)
63
+
64
+ def is_reasoning_end_streaming(
65
+ self, input_ids: Sequence[int], delta_ids: Iterable[int]
66
+ ) -> bool:
67
+ if self._identity_parser is not None:
68
+ return self._identity_parser.is_reasoning_end_streaming(
69
+ input_ids, delta_ids
70
+ )
71
+
72
+ return super().is_reasoning_end_streaming(input_ids, delta_ids)
73
+
74
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
75
+ if self._identity_parser is not None:
76
+ return self._identity_parser.extract_content_ids(input_ids)
77
+
78
+ return super().extract_content_ids(input_ids)
79
+
80
+ def extract_reasoning(
81
+ self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest"
82
+ ) -> tuple[str | None, str | None]:
83
+ if self._identity_parser is not None:
84
+ return self._identity_parser.extract_reasoning(model_output, request)
85
+
86
+ return super().extract_reasoning(model_output, request)
87
+
88
+ def extract_reasoning_streaming(
89
+ self,
90
+ previous_text: str,
91
+ current_text: str,
92
+ delta_text: str,
93
+ previous_token_ids: Sequence[int],
94
+ current_token_ids: Sequence[int],
95
+ delta_token_ids: Sequence[int],
96
+ ) -> DeltaMessage | None:
97
+ if self._identity_parser is not None:
98
+ return self._identity_parser.extract_reasoning_streaming(
99
+ previous_text,
100
+ current_text,
101
+ delta_text,
102
+ previous_token_ids,
103
+ current_token_ids,
104
+ delta_token_ids,
105
+ )
106
+
107
+ ret = super().extract_reasoning_streaming(
108
+ previous_text,
109
+ current_text,
110
+ delta_text,
111
+ previous_token_ids,
112
+ current_token_ids,
113
+ delta_token_ids,
114
+ )
115
+ if (
116
+ ret is not None
117
+ and self.start_token_id not in previous_token_ids
118
+ and self.start_token_id not in delta_token_ids
119
+ ):
120
+ if self.end_token_id in delta_token_ids:
121
+ # end token in delta with more tokens,
122
+ # extract reasoning content and content
123
+ end_index = delta_text.find(self.end_token)
124
+ reasoning = delta_text[:end_index]
125
+ content = delta_text[end_index + len(self.end_token) :]
126
+ return DeltaMessage(
127
+ reasoning=reasoning,
128
+ content=content if content else None,
129
+ )
130
+ elif self.end_token_id in previous_token_ids:
131
+ # end token in previous, thinking content ends
132
+ return DeltaMessage(content=delta_text)
133
+ else:
134
+ # no end token in previous or delta, reasoning content continues
135
+ return DeltaMessage(reasoning=delta_text)
136
+
137
+ return ret
vllm/tool_parsers/__init__.py
+4
@@ -66,6 +66,10 @@ _TOOL_PARSERS_TO_REGISTER = {
66
"hunyuan_a13b_tool_parser",
67
"HunyuanA13BToolParser",
68
),
69
+ "hy_v3": (
70
+ "hy_v3_tool_parser",
71
+ "HYV3ToolParser",
72
+ ),
73
"internlm": (
74
"internlm2_tool_parser",
75
"Internlm2ToolParser",
vllm/tool_parsers/hy_v3_tool_parser.py
new
+645
@@ -0,0 +1,645 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import ast
5
+import json
6
+from collections.abc import Sequence
7
+from typing import Any
8
+
9
+import regex as re
10
+
11
+from vllm.entrypoints.chat_utils import make_tool_call_id
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
+ ChatCompletionRequest,
14
+ ChatCompletionToolsParam,
15
+)
16
+from vllm.entrypoints.openai.engine.protocol import (
17
+ DeltaFunctionCall,
18
+ DeltaMessage,
19
+ DeltaToolCall,
20
+ ExtractedToolCallInformation,
21
+ FunctionCall,
22
+ ToolCall,
23
+)
24
+from vllm.logger import init_logger
25
+from vllm.tokenizers import TokenizerLike
26
+from vllm.tool_parsers.abstract_tool_parser import (
27
+ Tool,
28
+ ToolParser,
29
+)
30
+
31
+logger = init_logger(__name__)
32
+
33
+
34
+class HYV3ToolParser(ToolParser):
35
+ _TYPE_ALIASES: dict[str, str] = {
36
+ "str": "string",
37
+ "text": "string",
38
+ "varchar": "string",
39
+ "char": "string",
40
+ "enum": "string",
41
+ "bool": "boolean",
42
+ "binary": "boolean",
43
+ "int": "integer",
44
+ "float": "number",
45
+ "double": "number",
46
+ "list": "array",
47
+ "dict": "object",
48
+ "map": "object",
49
+ }
50
+
51
+ # Prefix-based wildcard matching for non-standard type names.
52
+ # Following the same approach as
53
+ # qwen3coder_tool_parser._convert_param_value which uses
54
+ # param_type.startswith("int"), startswith("uint"), etc.
55
+ _INTEGER_PREFIXES: tuple[str, ...] = (
56
+ "int",
57
+ "uint",
58
+ "long",
59
+ "short",
60
+ "unsigned",
61
+ )
62
+ _NUMBER_PREFIXES: tuple[str, ...] = ("num", "float")
63
+
64
+ @staticmethod
65
+ def _normalize_type(raw_type: str) -> str:
66
+ """Map non-standard type aliases to JSON Schema standard names.
67
+
68
+ First performs exact lookup in _TYPE_ALIASES. On miss, falls back
69
+ to prefix-based matching using startswith()
70
+ - int*/uint*/long*/short*/unsigned* → "integer"
71
+ - num*/float* → "number"
72
+ """
73
+ exact = HYV3ToolParser._TYPE_ALIASES.get(raw_type)
74
+ if exact is not None:
75
+ return exact
76
+ lower = raw_type.lower()
77
+ if any(lower.startswith(p) for p in HYV3ToolParser._INTEGER_PREFIXES):
78
+ return "integer"
79
+ if any(lower.startswith(p) for p in HYV3ToolParser._NUMBER_PREFIXES):
80
+ return "number"
81
+ return raw_type
82
+
83
+ @staticmethod
84
+ def _get_arg_schema(
85
+ function_name: str,
86
+ arg_key: str,
87
+ tools: list[ChatCompletionToolsParam] | None,
88
+ ) -> dict:
89
+ """Look up a specific argument's property schema from the tools list."""
90
+ if tools is None:
91
+ return {}
92
+ for tool in tools:
93
+ if tool.function.name == function_name:
94
+ if tool.function.parameters is None:
95
+ return {}
96
+ return tool.function.parameters.get("properties", {}).get(arg_key, {})
97
+ logger.warning("No tool named '%s'.", function_name)
98
+ return {}
99
+
100
+ @staticmethod
101
+ def _get_schema_options(arg_schema: dict) -> list[dict]:
102
+ """Normalize any property schema into a list of sub-schemas.
103
+ - has type (single type) → return [arg_schema]
104
+ - anyOf → return the anyOf list
105
+ - oneOf → return the oneOf list
106
+ - fallback → [{"type": "string"}]
107
+
108
+ Note: single ``type`` has the highest priority.
109
+ """
110
+ if "type" in arg_schema:
111
+ return [arg_schema]
112
+ if "anyOf" in arg_schema:
113
+ return arg_schema["anyOf"]
114
+ if "oneOf" in arg_schema:
115
+ return arg_schema["oneOf"]
116
+
117
+ return [{"type": "string"}]
118
+
119
+ @staticmethod
120
+ def _get_types(arg_schema: dict) -> set[str]:
121
+ """Extract normalized, non-null type set from a property schema."""
122
+ schemas = HYV3ToolParser._get_schema_options(arg_schema)
123
+ return {
124
+ HYV3ToolParser._normalize_type(s.get("type", "string")) for s in schemas
125
+ } - {"null"}
126
+
127
+ @staticmethod
128
+ def _is_only_string_type(
129
+ function_name: str,
130
+ arg_key: str,
131
+ tools: list[ChatCompletionToolsParam] | None,
132
+ ) -> bool:
133
+ """Return True if the parameter's type set is exactly {"string"}.
134
+
135
+ Only pure string types get partial value streaming; compound types
136
+ like anyOf(string | array) do not, since the partial value might
137
+ end up being a JSON array or object.
138
+ """
139
+ arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools)
140
+ types = HYV3ToolParser._get_types(arg_schema)
141
+ return types == {"string"}
142
+
143
+ @staticmethod
144
+ def _try_parse_bool(value: str) -> bool | None:
145
+ """Try to parse a string as bool; return None on failure."""
146
+ lower = value.lower()
147
+ if lower == "true":
148
+ return True
149
+ elif lower == "false":
150
+ return False
151
+ return None
152
+
153
+ @staticmethod
154
+ def _try_parse_int(value: str) -> int | None:
155
+ """Try to parse a string as int; return None on failure."""
156
+ try:
157
+ return int(value)
158
+ except (ValueError, TypeError):
159
+ return None
160
+
161
+ @staticmethod
162
+ def _try_parse_wildcard_number(value: str) -> int | float | None:
163
+ """Try to parse a string as a number (int or float).
164
+
165
+ Decision rule: if the string contains '.' or 'e'/'E' (scientific
166
+ notation), parse as float; otherwise parse as int.
167
+
168
+ Examples:
169
+ "5" → int(5)
170
+ "5.0" → float(5.0)
171
+ "5.3" → float(5.3)
172
+ "1e3" → float(1000.0)
173
+ "-3" → int(-3)
174
+
175
+ Return None on failure.
176
+ """
177
+ try:
178
+ if "." in value or "e" in value or "E" in value:
179
+ return float(value)
180
+ return int(value)
181
+ except (ValueError, TypeError):
182
+ return None
183
+
184
+ @staticmethod
185
+ def _deserialize(value: str) -> Any:
186
+ """Deserialize a string value using json.loads then ast.literal_eval."""
187
+ try:
188
+ return json.loads(value)
189
+ except Exception:
190
+ pass
191
+ try:
192
+ return ast.literal_eval(value)
193
+ except Exception:
194
+ pass
195
+ return value
196
+
197
+ @staticmethod
198
+ def _parse_value(
199
+ value: str,
200
+ function_name: str,
201
+ arg_key: str,
202
+ tools: list[ChatCompletionToolsParam] | None,
203
+ ) -> Any:
204
+ """Unified argument value parser with anyOf/oneOf support.
205
+
206
+ Fallthrough chain:
207
+ bool → int → number(wildcard_number)
208
+ → json.loads for array/object
209
+ → string → _deserialize
210
+ """
211
+ arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools)
212
+ types = HYV3ToolParser._get_types(arg_schema)
213
+
214
+ # 1. Try bool
215
+ if "boolean" in types:
216
+ result_bool = HYV3ToolParser._try_parse_bool(value)
217
+ if result_bool is not None:
218
+ return result_bool
219
+
220
+ # 2. Try int
221
+ if "integer" in types:
222
+ result_int = HYV3ToolParser._try_parse_int(value)
223
+ if result_int is not None:
224
+ return result_int
225
+
226
+ # 3. Try number (wildcard_number: int if no '.'/e/E, float otherwise)
227
+ if "number" in types:
228
+ result_number = HYV3ToolParser._try_parse_wildcard_number(value)
229
+ if result_number is not None:
230
+ return result_number
231
+
232
+ # 4. Try json.loads (covers array/object and other unlisted types)
233
+ if types - {"string", "boolean", "integer", "number"}:
234
+ try:
235
+ return json.loads(value)
236
+ except (json.JSONDecodeError, ValueError):
237
+ pass
238
+
239
+ # 5. String fallback
240
+ if "string" in types:
241
+ return value
242
+
243
+ # 6. Final fallback
244
+ return HYV3ToolParser._deserialize(value)
245
+
246
+ def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
247
+ super().__init__(tokenizer, tools)
248
+
249
+ self.current_tool_name_sent: bool = False
250
+ self.prev_tool_call_arr: list[dict] = []
251
+ self.current_tool_id: int = -1
252
+ self.streamed_args_for_tool: list[
253
+ str
254
+ ] = [] # map what has been streamed for each tool so far to a list
255
+
256
+ # Streaming state: send tool name first, then return arguments at once
257
+ self._streaming_tool_name: str | None = None # tool name being streamed
258
+
259
+ # State fields for incremental argument streaming
260
+ self._completed_args: dict = {} # closed {key: parsed_value}
261
+ self._current_arg_key: str | None = None # key being collected
262
+ self._current_arg_is_string: bool = False # is current arg pure string?
263
+ self._streamed_json_len: int = 0 # bytes of JSON already sent
264
+
265
+ self.tool_calls_start_token: str = "<tool_calls>"
266
+ self.tool_calls_end_token: str = "</tool_calls>"
267
+
268
+ self.tool_call_start_token: str = "<tool_call>"
269
+ self.tool_call_end_token: str = "</tool_call>"
270
+
271
+ self.tool_sep_token: str = "<tool_sep>"
272
+
273
+ self.arg_key_start_token: str = "<arg_key>"
274
+ self.arg_key_end_token: str = "</arg_key>"
275
+
276
+ self.arg_value_start_token: str = "<arg_value>"
277
+ self.arg_value_end_token: str = "</arg_value>"
278
+
279
+ self.tool_call_regex = re.compile(
280
+ rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}"
281
+ rf"(.*?){self.tool_call_end_token}",
282
+ re.DOTALL,
283
+ )
284
+
285
+ self.tool_call_portion_regex = re.compile(
286
+ rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}(.*)", re.DOTALL
287
+ )
288
+
289
+ self.func_args_regex = re.compile(
290
+ rf"{self.arg_key_start_token}(.*?){self.arg_key_end_token}\s*"
291
+ rf"{self.arg_value_start_token}(.*?){self.arg_value_end_token}",
292
+ re.DOTALL,
293
+ )
294
+
295
+ if not self.model_tokenizer:
296
+ raise ValueError(
297
+ "The model tokenizer must be passed to the ToolParser "
298
+ "constructor during construction."
299
+ )
300
+ self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token)
301
+ self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token)
302
+
303
+ self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
304
+ self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
305
+ self._buffer = ""
306
+
307
+ if (
308
+ self.tool_calls_start_token_id is None
309
+ or self.tool_calls_end_token_id is None
310
+ ):
311
+ raise RuntimeError(
312
+ "HYV3 Tool parser could not locate tool call "
313
+ "start/end tokens in the tokenizer!"
314
+ )
315
+
316
+ def _extract_tool_calls(
317
+ self,
318
+ model_output: str,
319
+ request: ChatCompletionRequest,
320
+ ) -> list[ToolCall]:
321
+ try:
322
+ function_call_tuples = []
323
+ # start_token{name}sep_token{args}end_token...
324
+ function_calls = self.tool_call_regex.findall(model_output)
325
+ if function_calls:
326
+ function_call_tuples.extend(function_calls)
327
+ remaining = model_output.split(self.tool_call_end_token)[-1]
328
+ function_calls = self.tool_call_portion_regex.findall(remaining)
329
+ function_call_tuples += function_calls
330
+ else:
331
+ function_calls = self.tool_call_portion_regex.findall(model_output)
332
+ if function_calls:
333
+ function_call_tuples.extend(function_calls)
334
+ tool_calls = []
335
+ for match in function_call_tuples:
336
+ function_name, function_args = match
337
+ function_name = function_name.strip()
338
+ function_args = function_args.strip()
339
+
340
+ arg_pairs = self.func_args_regex.findall(function_args)
341
+ arg_dict = {}
342
+ for key, value in arg_pairs:
343
+ parsed_value = HYV3ToolParser._parse_value(
344
+ value, function_name, key, request.tools
345
+ )
346
+ arg_dict[key] = parsed_value
347
+ tool_calls.append(
348
+ ToolCall(
349
+ type="function",
350
+ function=FunctionCall(
351
+ name=function_name,
352
+ arguments=json.dumps(arg_dict, ensure_ascii=False),
353
+ ),
354
+ )
355
+ )
356
+ return tool_calls
357
+ except Exception:
358
+ logger.exception("Error in extracting tool call from response.")
359
+ return []
360
+
361
+ def extract_tool_calls(
362
+ self,
363
+ model_output: str,
364
+ request: ChatCompletionRequest,
365
+ ) -> ExtractedToolCallInformation:
366
+ # sanity check; avoid unnecessary processing
367
+ if self.tool_calls_start_token not in model_output:
368
+ return ExtractedToolCallInformation(
369
+ tools_called=False, tool_calls=[], content=model_output
370
+ )
371
+ else:
372
+ try:
373
+ tool_calls = self._extract_tool_calls(model_output, request)
374
+
375
+ s_index = model_output.find(self.tool_calls_start_token)
376
+ content = model_output[:s_index] if s_index != -1 else model_output
377
+ return ExtractedToolCallInformation(
378
+ tools_called=True,
379
+ tool_calls=tool_calls,
380
+ content=content if content else None,
381
+ )
382
+
383
+ except Exception:
384
+ logger.exception("Error in extracting tool call from response.")
385
+ return ExtractedToolCallInformation(
386
+ tools_called=False, tool_calls=[], content=model_output
387
+ )
388
+
389
+ def _reset_streaming_tool_state(self):
390
+ """Reset the streaming state for a single tool call."""
391
+ self._streaming_tool_name = None
392
+ self._completed_args = {}
393
+ self._current_arg_key = None
394
+ self._current_arg_is_string = False
395
+ self._streamed_json_len = 0
396
+
397
+ def extract_tool_calls_streaming(
398
+ self,
399
+ previous_text: str,
400
+ current_text: str,
401
+ delta_text: str,
402
+ previous_token_ids: Sequence[int],
403
+ current_token_ids: Sequence[int],
404
+ delta_token_ids: Sequence[int],
405
+ request: ChatCompletionRequest,
406
+ ) -> DeltaMessage | None:
407
+ # Check whether current tokens contain the tool_calls start token
408
+ if self.tool_calls_start_token_id not in current_token_ids:
409
+ return DeltaMessage(content=delta_text)
410
+
411
+ # Encountered tool_calls start tag; extract preceding content and buffer
412
+ if self.tool_calls_start_token in delta_text:
413
+ text_parts = delta_text.split(self.tool_calls_start_token)
414
+ self._buffer += text_parts[-1]
415
+ if text_parts[0]:
416
+ return DeltaMessage(content=text_parts[0])
417
+ # Don't return None; continue processing buffer for complete content
418
+ else:
419
+ self._buffer += delta_text
420
+
421
+ # Encountered finish, extract valid arguments
422
+ if (
423
+ current_text.find(self.tool_call_end_token + self.tool_calls_end_token)
424
+ != -1
425
+ and self._buffer.find(self.tool_call_end_token) == -1
426
+ ):
427
+ self._buffer += self.tool_call_end_token + self.tool_calls_end_token
428
+
429
+ cur_text = self._buffer
430
+
431
+ # Haven't encountered tool_call start tag yet; keep buffering
432
+ start_idx = cur_text.find(self.tool_call_start_token)
433
+ if start_idx == -1 and self._streaming_tool_name is None:
434
+ self._buffer = ""
435
+ return None
436
+
437
+ # === Phase 1: Detect tool name (send when tool_sep_token is seen) ===
438
+ name_delta: DeltaMessage | None = None
439
+ if self._streaming_tool_name is None:
440
+ sep_idx = cur_text.find(self.tool_sep_token)
441
+ if sep_idx == -1:
442
+ # tool_sep not yet seen; keep buffering from tool_call_start
443
+ self._buffer = cur_text[start_idx:]
444
+ return None
445
+
446
+ # Extract tool name: between tool_call_start_token and tool_sep_token
447
+ name_start = start_idx + len(self.tool_call_start_token)
448
+ tool_name = cur_text[name_start:sep_idx].strip()
449
+ self._streaming_tool_name = tool_name
450
+
451
+ # Update buffer: keep only content after tool_sep (i.e. the args portion)
452
+ self._buffer = cur_text[sep_idx + len(self.tool_sep_token) :]
453
+
454
+ # Increment tool_id and send a chunk containing only the name
455
+ self.current_tool_id += 1
456
+ self._current_tool_call_id = make_tool_call_id()
457
+ name_delta = DeltaMessage(
458
+ tool_calls=[
459
+ DeltaToolCall(
460
+ index=self.current_tool_id,
461
+ id=self._current_tool_call_id,
462
+ type="function",
463
+ function=DeltaFunctionCall(
464
+ name=tool_name,
465
+ ),
466
+ )
467
+ ]
468
+ )
469
+
470
+ # Check if buffer already has complete arguments (all-in-one-delta)
471
+ if self.tool_call_end_token not in self._buffer:
472
+ return name_delta
473
+ # Buffer already has a complete tool call; continue to phase 2 below
474
+
475
+ # === Phase 2: Incremental argument streaming ===
476
+ return self._extract_streaming_incremental(name_delta, request)
477
+
478
+ def _make_args_delta(self, argument_diff: str) -> DeltaMessage:
479
+ """Build a DeltaMessage containing only an arguments diff."""
480
+ return DeltaMessage(
481
+ tool_calls=[
482
+ DeltaToolCall(
483
+ index=self.current_tool_id,
484
+ function=DeltaFunctionCall(arguments=argument_diff),
485
+ )
486
+ ]
487
+ )
488
+
489
+ def _extract_streaming_incremental(
490
+ self,
491
+ name_delta: DeltaMessage | None,
492
+ request: ChatCompletionRequest,
493
+ ) -> DeltaMessage | None:
494
+ """Incremental phase-2: scan tags in buffer, emit JSON diffs.
495
+
496
+ Strategy:
497
+ - Track completed args and emit each one as a JSON fragment.
498
+ - For string-typed args, stream the value character-by-character.
499
+ - Withhold the closing ``}`` until ``</tool_call>`` is seen.
500
+
501
+ We build JSON manually via fragments rather than using json.dumps
502
+ with a cursor, because json.dumps of partial-vs-full string values
503
+ produces incompatible prefixes (e.g. ``""}`` vs ``"Hello"}``).
504
+ """
505
+ buf = self._buffer
506
+ is_complete = self.tool_call_end_token in buf
507
+
508
+ if is_complete:
509
+ end_idx = buf.find(self.tool_call_end_token)
510
+ args_text = buf[:end_idx]
511
+ remaining = buf[end_idx + len(self.tool_call_end_token) :]
512
+ else:
513
+ args_text = buf
514
+ remaining = ""
515
+
516
+ # --- scan all fully closed kv pairs ---
517
+ arg_pairs = self.func_args_regex.findall(args_text)
518
+ for key, value in arg_pairs:
519
+ key = key.strip()
520
+ if key not in self._completed_args:
521
+ parsed_value = HYV3ToolParser._parse_value(
522
+ value, self._streaming_tool_name or "", key, request.tools
523
+ )
524
+ self._completed_args[key] = parsed_value
525
+
526
+ # --- detect partial (unclosed) kv at the tail ---
527
+ last_closed_end = 0
528
+ for m in self.func_args_regex.finditer(args_text):
529
+ last_closed_end = m.end()
530
+ tail = args_text[last_closed_end:]
531
+
532
+ partial_key: str | None = None
533
+ partial_value: str | None = None
534
+
535
+ ak_start = tail.find(self.arg_key_start_token)
536
+ if ak_start != -1:
537
+ ak_end = tail.find(
538
+ self.arg_key_end_token,
539
+ ak_start + len(self.arg_key_start_token),
540
+ )
541
+ if ak_end != -1:
542
+ partial_key = tail[
543
+ ak_start + len(self.arg_key_start_token) : ak_end
544
+ ].strip()
545
+ self._current_arg_key = partial_key
546
+ self._current_arg_is_string = HYV3ToolParser._is_only_string_type(
547
+ self._streaming_tool_name or "",
548
+ partial_key,
549
+ request.tools,
550
+ )
551
+
552
+ av_start = tail.find(self.arg_value_start_token, ak_end)
553
+ if av_start != -1:
554
+ val_content_start = av_start + len(self.arg_value_start_token)
555
+ if self._current_arg_is_string:
556
+ partial_value = tail[val_content_start:]
557
+ else:
558
+ # key not yet closed
559
+ self._current_arg_key = None
560
+ self._current_arg_is_string = False
561
+
562
+ # --- build the current JSON snapshot as a string ---
563
+ # We construct JSON manually so we can precisely control
564
+ # what gets sent incrementally.
565
+ snapshot_parts: list[str] = []
566
+ for k, v in self._completed_args.items():
567
+ k_json = json.dumps(k, ensure_ascii=False)
568
+ v_json = json.dumps(v, ensure_ascii=False)
569
+ snapshot_parts.append(f"{k_json}: {v_json}")
570
+
571
+ if partial_key is not None and partial_value is not None:
572
+ k_json = json.dumps(partial_key, ensure_ascii=False)
573
+ # For string partial value, we build the JSON string
574
+ # WITHOUT the closing quote, so the prefix stays stable
575
+ # as the value grows. The closing `"` and `}` will be
576
+ # sent when the value or tool_call closes.
577
+ escaped_val = (
578
+ partial_value.replace("\\", "\\\\")
579
+ .replace('"', '\\"')
580
+ .replace("\n", "\\n")
581
+ .replace("\r", "\\r")
582
+ .replace("\t", "\\t")
583
+ )
584
+ # Note: no closing " here – it's appended only on close
585
+ snapshot_parts.append(f'{k_json}: "{escaped_val}')
586
+
587
+ snapshot = "{" + ", ".join(snapshot_parts) + "}"
588
+
589
+ # --- compute diff ---
590
+ argument_diff: str | None = None
591
+
592
+ if is_complete:
593
+ # Tool call finished – send everything remaining.
594
+ # Build final snapshot with proper JSON (all values closed).
595
+ final_args = dict(self._completed_args)
596
+ final_json = json.dumps(final_args, ensure_ascii=False)
597
+ if self._streamed_json_len < len(final_json):
598
+ argument_diff = final_json[self._streamed_json_len :]
599
+ self._streamed_json_len = len(final_json)
600
+
601
+ # Record into prev_tool_call_arr
602
+ self.prev_tool_call_arr.append(
603
+ {
604
+ "name": self._streaming_tool_name,
605
+ "arguments": final_args,
606
+ }
607
+ )
608
+ self.streamed_args_for_tool.append(final_json)
609
+
610
+ self._reset_streaming_tool_state()
611
+ self._buffer = remaining
612
+ else:
613
+ # Still in progress – withhold the tail.
614
+ # For open strings: snapshot ends with ...partial_val}
615
+ # we withhold "}" (1 char) – the missing closing " will
616
+ # be sent when the value closes.
617
+ # For no open string: snapshot ends with ...value"}
618
+ # we withhold "}" (1 char).
619
+ end = len(snapshot) - 1 # exclude trailing "}"
620
+ if end > self._streamed_json_len:
621
+ argument_diff = snapshot[self._streamed_json_len : end]
622
+ self._streamed_json_len = end
623
+
624
+ # --- construct return DeltaMessage ---
625
+ if name_delta is not None and argument_diff:
626
+ nd_func = name_delta.tool_calls[0].function
627
+ return DeltaMessage(
628
+ tool_calls=[
629
+ DeltaToolCall(
630
+ index=self.current_tool_id,
631
+ id=self._current_tool_call_id,
632
+ type="function",
633
+ function=DeltaFunctionCall(
634
+ name=nd_func.name if nd_func else None,
635
+ arguments=argument_diff,
636
+ ),
637
+ )
638
+ ]
639
+ )
640
+ elif name_delta is not None:
641
+ return name_delta
642
+ elif argument_diff:
643
+ return self._make_args_delta(argument_diff)
644
+ else:
645
+ return None
vllm/transformers_utils/config.py
+1
@@ -94,6 +94,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
94
funaudiochat="FunAudioChatConfig",
95
granite4_vision="Granite4VisionConfig",
96
hunyuan_vl="HunYuanVLConfig",
97
+ hy_v3="HYV3Config",
98
isaac="IsaacConfig",
99
kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3
100
kimi_linear="KimiLinearConfig",
vllm/transformers_utils/configs/__init__.py
+2
@@ -36,6 +36,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
36
"HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
37
"HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
38
"HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
39
+ "HYV3Config": "vllm.transformers_utils.configs.hy_v3",
40
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
41
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
42
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -97,6 +98,7 @@ __all__ = [
98
"HunYuanVLConfig",
99
"HunYuanVLTextConfig",
100
"HunYuanVLVisionConfig",
101
+ "HYV3Config",
102
"HyperCLOVAXConfig",
103
"IsaacConfig",
104
"RWConfig",
vllm/transformers_utils/configs/hy_v3.py
new
+185
@@ -0,0 +1,185 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
+
5
+from transformers.configuration_utils import PretrainedConfig
6
+
7
+
8
+class HYV3Config(PretrainedConfig):
9
+ r"""
10
+ This is the configuration class to store the configuration of a [`HYV3Model`].
11
+ It is used to instantiate a HYV3 model (HY V3 MoE language model) according to
12
+ the specified arguments.
13
+
14
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to
15
+ control the model outputs. Read the documentation from [`PretrainedConfig`]
16
+ for more information.
17
+
18
+ Args:
19
+ vocab_size (`int`, *optional*, defaults to 120832):
20
+ Vocabulary size of the model.
21
+ hidden_size (`int`, *optional*, defaults to 4096):
22
+ Dimension of the hidden representations.
23
+ intermediate_size (`int`, *optional*, defaults to 13312):
24
+ Dimension of the dense FFN intermediate representations.
25
+ num_hidden_layers (`int`, *optional*, defaults to 80):
26
+ Number of hidden layers in the Transformer decoder.
27
+ num_attention_heads (`int`, *optional*, defaults to 64):
28
+ Number of attention heads for each attention layer.
29
+ num_key_value_heads (`int`, *optional*, defaults to 8):
30
+ Number of key-value heads for grouped-query attention.
31
+ head_dim (`int`, *optional*, defaults to 128):
32
+ Dimension per attention head.
33
+ hidden_act (`str`, *optional*, defaults to `"silu"`):
34
+ Activation function used in FFN layers.
35
+ max_position_embeddings (`int`, *optional*, defaults to 131072):
36
+ Maximum sequence length supported by the model.
37
+ initializer_range (`float`, *optional*, defaults to 0.006):
38
+ Standard deviation of the truncated normal initializer for weight
39
+ initialization.
40
+ rms_norm_eps (`float`, *optional*, defaults to 1e-5):
41
+ Epsilon for RMS normalization layers.
42
+ use_cache (`bool`, *optional*, defaults to `True`):
43
+ Whether to use KV cache for decoding.
44
+ pad_token_id (`int`, *optional*):
45
+ Padding token id.
46
+ bos_token_id (`int`, *optional*):
47
+ Beginning-of-sequence token id.
48
+ eos_token_id (`int` or `List[int]`, *optional*):
49
+ End-of-sequence token id(s).
50
+ rope_parameters (`dict`, *optional*):
51
+ The parameters of the RoPE embeddings.
52
+ qk_norm (`bool`, *optional*, defaults to `True`):
53
+ Whether to apply RMSNorm to query and key states before attention.
54
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
55
+ Whether to tie input and output embedding weights.
56
+ enable_attention_fp32_softmax (`bool`, *optional*, defaults to `False`):
57
+ Whether to upcast attention softmax to float32. Note: the eager attention
58
+ path always computes softmax in float32 regardless of this setting; this
59
+ flag is reserved for future use with custom attention backends.
60
+ enable_lm_head_fp32 (`bool`, *optional*, defaults to `True`):
61
+ Whether to upcast the LM head computation to float32.
62
+ num_experts (`int`, *optional*, defaults to 192):
63
+ Total number of MoE experts.
64
+ num_experts_per_tok (`int`, *optional*, defaults to 8):
65
+ Number of experts selected per token (top-k routing).
66
+ num_shared_experts (`int`, *optional*, defaults to 1):
67
+ Number of always-active shared experts combined into a single MLP.
68
+ expert_hidden_dim (`int`, *optional*, defaults to 1536):
69
+ Intermediate dimension of each individual MoE expert.
70
+ moe_router_enable_expert_bias (`bool`, *optional*, defaults to `True`):
71
+ Whether to use per-expert load-balancing bias in the router.
72
+ moe_router_use_sigmoid (`bool`, *optional*, defaults to `True`):
73
+ Whether to use sigmoid (instead of softmax) for router scoring.
74
+ route_norm (`bool`, *optional*, defaults to `True`):
75
+ Whether to normalize routing scores when using sigmoid routing.
76
+ router_scaling_factor (`float`, *optional*):
77
+ Optional multiplicative scaling factor applied to routing scores.
78
+ use_grouped_mm (`bool`, *optional*, defaults to `False`):
79
+ Whether to use grouped GEMM for expert computation (not yet implemented).
80
+ enable_moe_fp32_combine (`bool`, *optional*, defaults to `False`):
81
+ Whether to accumulate expert outputs in float32.
82
+ first_k_dense_replace (`int`, *optional*, defaults to 1):
83
+ Number of initial decoder layers that use a dense FFN instead of MoE.
84
+ output_router_logits (`bool`, *optional*, defaults to `False`):
85
+ Whether to output router logits from each MoE layer. Useful for computing
86
+ auxiliary load-balancing loss during training. Disabled by default to avoid
87
+ the memory overhead of storing per-layer router tensors during inference.
88
+
89
+ Example:
90
+ ```python
91
+ >>> from transformers import HYV3Config, HYV3Model
92
+
93
+ >>> config = HYV3Config()
94
+ >>> model = HYV3Model(config)
95
+ ```
96
+ """
97
+
98
+ model_type = "hy_v3"
99
+ keys_to_ignore_at_inference = ["past_key_values"]
100
+
101
+ def __init__(
102
+ self,
103
+ vocab_size=120832,
104
+ hidden_size=4096,
105
+ intermediate_size=13312,
106
+ num_hidden_layers=80,
107
+ num_attention_heads=64,
108
+ num_key_value_heads=8,
109
+ head_dim=128,
110
+ hidden_act="silu",
111
+ max_position_embeddings=131072,
112
+ initializer_range=0.006,
113
+ rms_norm_eps=1e-5,
114
+ use_cache=True,
115
+ pad_token_id=None,
116
+ bos_token_id=None,
117
+ eos_token_id=None,
118
+ rope_parameters: dict[str, Any] | None = None,
119
+ qk_norm=True,
120
+ tie_word_embeddings=False,
121
+ enable_attention_fp32_softmax=False,
122
+ enable_lm_head_fp32=True,
123
+ # MoE specific
124
+ num_experts=192,
125
+ num_experts_per_tok=8,
126
+ num_shared_experts=1,
127
+ expert_hidden_dim=1536,
128
+ moe_router_enable_expert_bias=True,
129
+ moe_router_use_sigmoid=True,
130
+ route_norm=True,
131
+ router_scaling_factor=None,
132
+ use_grouped_mm=False,
133
+ enable_moe_fp32_combine=False,
134
+ # Dense/MoE layer control
135
+ first_k_dense_replace=1,
136
+ output_router_logits=False,
137
+ **kwargs,
138
+ ):
139
+ self.vocab_size = vocab_size
140
+ self.hidden_size = hidden_size
141
+ self.intermediate_size = intermediate_size
142
+ self.num_hidden_layers = num_hidden_layers
143
+ self.num_attention_heads = num_attention_heads
144
+ self.num_key_value_heads = num_key_value_heads
145
+ self.head_dim = head_dim
146
+ self.hidden_act = hidden_act
147
+ self.max_position_embeddings = max_position_embeddings
148
+ self.initializer_range = initializer_range
149
+ self.rms_norm_eps = rms_norm_eps
150
+ self.use_cache = use_cache
151
+ rope_theta = kwargs.pop("rope_theta", 11158840.0)
152
+ if rope_parameters is None:
153
+ rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
154
+ self.rope_parameters = rope_parameters
155
+ self.qk_norm = qk_norm
156
+ self.tie_word_embeddings = tie_word_embeddings
157
+ self.enable_lm_head_fp32 = enable_lm_head_fp32
158
+ self.enable_attention_fp32_softmax = enable_attention_fp32_softmax
159
+
160
+ # MoE specific
161
+ self.num_experts = num_experts
162
+ self.num_experts_per_tok = num_experts_per_tok
163
+ self.num_shared_experts = num_shared_experts
164
+ self.expert_hidden_dim = expert_hidden_dim
165
+ self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
166
+ self.moe_router_use_sigmoid = moe_router_use_sigmoid
167
+ self.route_norm = route_norm
168
+ self.use_grouped_mm = use_grouped_mm
169
+ self.router_scaling_factor = router_scaling_factor
170
+ self.enable_moe_fp32_combine = enable_moe_fp32_combine
171
+
172
+ # Dense/MoE layer control
173
+ self.first_k_dense_replace = first_k_dense_replace
174
+ self.output_router_logits = output_router_logits
175
+
176
+ if eos_token_id is not None and isinstance(eos_token_id, int):
177
+ eos_token_id = [eos_token_id]
178
+
179
+ super().__init__(
180
+ pad_token_id=pad_token_id,
181
+ bos_token_id=bos_token_id,
182
+ eos_token_id=eos_token_id,
183
+ tie_word_embeddings=tie_word_embeddings,
184
+ **kwargs,
185
+ )