@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5964069367
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[New Model] Add Seed-Oss model (#23241)

Signed-off-by: jiabin.00 <jiabin.00@bytedance.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>

Bin Jia committed Aug 22, 2025 at 12:58 UTC 5964069367a7d54c3816ce3faba79e02110cde17
7 files changed +1629
docs/models/supported_models.md
+1
@@ -401,6 +401,7 @@ th {
401 | `Qwen2MoeForCausalLM` | Qwen2MoE | `Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc. | ✅︎ | ✅︎ | ✅︎ |
402 | `Qwen3ForCausalLM` | Qwen3 | `Qwen/Qwen3-8B`, etc. | ✅︎ | ✅︎ | ✅︎ |
403 | `Qwen3MoeForCausalLM` | Qwen3MoE | `Qwen/Qwen3-30B-A3B`, etc. | ✅︎ | ✅︎ | ✅︎ |
404 +| `SeedOssForCausalLM` | SeedOss | `ByteDance-Seed/Seed-OSS-36B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
405 | `StableLmForCausalLM` | StableLM | `stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc. | | | ✅︎ |
406 | `Starcoder2ForCausalLM` | Starcoder2 | `bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc. | | ✅︎ | ✅︎ |
407 | `SolarForCausalLM` | Solar Pro | `upstage/solar-pro-preview-instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
tests/models/registry.py
+3
@@ -292,6 +292,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
292 "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"),
293 "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
294 "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"),
295 + "SeedOssForCausalLM": _HfExamplesInfo("ByteDance-Seed/Seed-OSS-36B-Instruct", # noqa: E501
296 + trust_remote_code=True,
297 + is_available_online=False),
298 "SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
299 "StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"), # noqa: E501
300 "StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
tests/tool_use/test_seed_oss_tool_parser.py new
+459
@@ -0,0 +1,459 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# ruff: noqa: E501
4 +
5 +import json
6 +from collections.abc import Generator
7 +from typing import Optional
8 +
9 +import pytest
10 +
11 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 + ChatCompletionToolsParam,
13 + DeltaMessage, FunctionCall,
14 + ToolCall)
15 +from vllm.entrypoints.openai.tool_parsers import SeedOssToolParser
16 +from vllm.transformers_utils.detokenizer import detokenize_incrementally
17 +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
18 +
19 +# Use a common model that is likely to be available
20 +MODEL = "ByteDance-Seed/Seed-OSS-36B-Instruct"
21 +
22 +
23 +@pytest.fixture(scope="module")
24 +def seed_oss_tokenizer():
25 + return get_tokenizer(tokenizer_name=MODEL, trust_remote_code=True)
26 +
27 +
28 +@pytest.fixture
29 +def seed_oss_tool_parser(seed_oss_tokenizer):
30 + return SeedOssToolParser(seed_oss_tokenizer)
31 +
32 +
33 +@pytest.fixture
34 +def sample_tools():
35 + return [
36 + ChatCompletionToolsParam(
37 + type="function",
38 + function={
39 + "name": "get_weather",
40 + "description": "Get current temperature for a given location.",
41 + "parameters": {
42 + "type": "object",
43 + "properties": {
44 + "location": {
45 + "type": "string",
46 + "description":
47 + "City and country e.g. Bogotá, Colombia"
48 + },
49 + "unit": {
50 + "type": "string",
51 + "description": "this is the unit of temperature"
52 + }
53 + },
54 + "required": ["location"],
55 + "additionalProperties": False
56 + },
57 + "returns": {
58 + "type": "object",
59 + "properties": {
60 + "temperature": {
61 + "type": "number",
62 + "description": "temperature in celsius"
63 + }
64 + },
65 + "required": ["temperature"],
66 + "additionalProperties": False
67 + },
68 + "strict": True
69 + }),
70 + ]
71 +
72 +
73 +def assert_tool_calls(actual_tool_calls: list[ToolCall],
74 + expected_tool_calls: list[ToolCall]):
75 + assert len(actual_tool_calls) == len(expected_tool_calls)
76 +
77 + for actual_tool_call, expected_tool_call in zip(actual_tool_calls,
78 + expected_tool_calls):
79 + # Seed-OSS tool call will not generate id
80 + assert actual_tool_call.type == "function"
81 + assert actual_tool_call.function == expected_tool_call.function
82 +
83 + assert actual_tool_call.function.name == expected_tool_call.function.name
84 + assert actual_tool_call.function.arguments == expected_tool_call.function.arguments
85 +
86 +
87 +def test_extract_tool_calls_no_tools(seed_oss_tool_parser):
88 + model_output = "This is a test response without any tool calls"
89 + extracted_tool_calls = seed_oss_tool_parser.extract_tool_calls(
90 + model_output, request=None) # type: ignore[arg-type]
91 +
92 + assert not extracted_tool_calls.tools_called
93 + assert extracted_tool_calls.tool_calls == []
94 + assert extracted_tool_calls.content == model_output
95 +
96 +
97 +@pytest.mark.parametrize(
98 + ids=[
99 + "tool_call_0_thinking_budget",
100 + "tool_call_512_thinkg_budget",
101 + "tool_call_unlimited_thinking_budget",
102 + ],
103 + argnames=["model_output", "expected_tool_calls", "expected_content"],
104 + argvalues=[
105 + ("""<seed:think>\n</seed:cot_budget_reflect>\n</seed:cot_budget_reflect>\n"""
106 + """The current thinking budget is 0, so I will directly start answering the question.\n</seed:think>\n"""
107 + """<seed:tool_call>\n<function=get_weather>\n"""
108 + """<parameter=location>Barcelona, Spain</parameter>\n</function>\n</seed:tool_call>""",
109 + [
110 + ToolCall(function=FunctionCall(
111 + name="get_weather",
112 + arguments=json.dumps({
113 + "location": "Barcelona, Spain",
114 + }, ),
115 + ),
116 + type='function')
117 + ],
118 + """<seed:think>\n</seed:cot_budget_reflect>\n</seed:cot_budget_reflect>\n"""
119 + """The current thinking budget is 0, so I will directly start answering the question.\n</seed:think>\n"""
120 + ),
121 + (
122 + """<seed:think>The user\'s current thinking budget is 512.</seed:cot_budget_reflect>\nLet me analyze the """
123 + """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """
124 + """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """
125 + """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """
126 + """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """
127 + """country). \n<seed:cot_budget_reflect>I have used 131 tokens, and there are 381 tokens remaining for use."""
128 + """</seed:cot_budget_reflect>\n Since the unit isn\'t specified, the function will default to Celsius, which """
129 + """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """
130 + """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """
131 + """user\'s input has a space, but the function might accept either; to be safe, using the standard format """
132 + """with a comma).\n<seed:cot_budget_reflect>I have used 257 tokens, and there are 255 tokens remaining for """
133 + """use.</seed:cot_budget_reflect>\n The unit parameter can be omitted since it\'s optional.</seed:think>\n"""
134 + """<seed:tool_call>\n<function=get_weather>\n<parameter=location>Barcelona, Spain</parameter>\n</function>"""
135 + """\n</seed:tool_call>""",
136 + [
137 + ToolCall(function=FunctionCall(
138 + name="get_weather",
139 + arguments=json.dumps({
140 + "location": "Barcelona, Spain",
141 + }, ),
142 + ),
143 + type='function')
144 + ],
145 + """<seed:think>The user\'s current thinking budget is 512.</seed:cot_budget_reflect>\nLet me analyze the """
146 + """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """
147 + """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """
148 + """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """
149 + """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """
150 + """country). \n<seed:cot_budget_reflect>I have used 131 tokens, and there are 381 tokens remaining for use."""
151 + """</seed:cot_budget_reflect>\n Since the unit isn\'t specified, the function will default to Celsius, which """
152 + """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """
153 + """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """
154 + """user\'s input has a space, but the function might accept either; to be safe, using the standard format """
155 + """with a comma).\n<seed:cot_budget_reflect>I have used 257 tokens, and there are 255 tokens remaining for """
156 + """use.</seed:cot_budget_reflect>\n The unit parameter can be omitted since it\'s optional.</seed:think>\n""",
157 + ),
158 + (
159 + """<seed:think>\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """
160 + """First, I need to remember the function I can use: get_weather. The function requires a """
161 + """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """
162 + """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """
163 + """let me check the function docstring again. Oh, the function says unit is optional, and """
164 + """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """
165 + """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """
166 + """The format is <seed:tool_call>\n<function=get_weather>\n<parameter=location>Barcelona, """
167 + """Spain</parameter>\n<parameter=unit>celsius</parameter>\n</function>\n</seed:tool_call>. """
168 + """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """
169 + """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """
170 + """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """
171 + """call should be as above. Then wait for the result to come back and tell the user the """
172 + """temperature in Celsius.</seed:think><seed:tool_call>\n<function=get_weather>\n<parameter=location>"""
173 + """Barcelona, Spain</parameter>\n<parameter=unit>celsius</parameter>\n</function>\n</seed:tool_call>""",
174 + [
175 + ToolCall(function=FunctionCall(
176 + name="get_weather",
177 + arguments=json.dumps(
178 + {
179 + "location": "Barcelona, Spain",
180 + "unit": "celsius",
181 + }, ),
182 + ),
183 + type='function')
184 + ],
185 + """<seed:think>\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """
186 + """First, I need to remember the function I can use: get_weather. The function requires a """
187 + """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """
188 + """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """
189 + """let me check the function docstring again. Oh, the function says unit is optional, and """
190 + """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """
191 + """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """
192 + """The format is <seed:tool_call>\n<function=get_weather>\n<parameter=location>Barcelona, """
193 + """Spain</parameter>\n<parameter=unit>celsius</parameter>\n</function>\n</seed:tool_call>. """
194 + """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """
195 + """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """
196 + """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """
197 + """call should be as above. Then wait for the result to come back and tell the user the """
198 + """temperature in Celsius.</seed:think>""",
199 + ),
200 + ],
201 +)
202 +def test_extract_tool_calls(seed_oss_tool_parser, sample_tools, model_output,
203 + expected_tool_calls, expected_content):
204 + request = ChatCompletionRequest(model=MODEL,
205 + messages=[],
206 + tools=sample_tools)
207 + extracted_tool_calls = seed_oss_tool_parser.extract_tool_calls(
208 + model_output, request=request) # type: ignore[arg-type]
209 + assert extracted_tool_calls.tools_called
210 +
211 + assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
212 +
213 + assert extracted_tool_calls.content == expected_content
214 +
215 +
216 +def test_streaming_tool_calls_no_tools(seed_oss_tool_parser):
217 + model_output = "This is a test response without any tool calls"
218 +
219 + result = seed_oss_tool_parser.extract_tool_calls_streaming(
220 + previous_text="his is a test response",
221 + current_text=model_output,
222 + delta_text=" without any tool calls.",
223 + previous_token_ids=[],
224 + current_token_ids=[],
225 + delta_token_ids=[],
226 + request=None,
227 + )
228 +
229 + # Should return the delta text as content
230 + assert result is not None
231 + assert hasattr(result, 'content')
232 + assert result.content == " without any tool calls."
233 +
234 +
235 +def stream_delta_message_generator(
236 + seed_oss_tool_parser: SeedOssToolParser,
237 + seed_oss_tokenizer: AnyTokenizer,
238 + model_output: str,
239 + request: Optional[ChatCompletionRequest] = None
240 +) -> Generator[DeltaMessage, None, None]:
241 + all_token_ids = seed_oss_tokenizer.encode(model_output,
242 + add_special_tokens=False)
243 +
244 + previous_text = ""
245 + previous_tokens = None
246 + prefix_offset = 0
247 + read_offset = 0
248 + for i, delta_token in enumerate(all_token_ids):
249 + delta_token_ids = [delta_token]
250 + previous_token_ids = all_token_ids[:i]
251 + current_token_ids = all_token_ids[:i + 1]
252 +
253 + (new_tokens, delta_text, new_prefix_offset,
254 + new_read_offset) = detokenize_incrementally(
255 + tokenizer=seed_oss_tokenizer,
256 + all_input_ids=current_token_ids,
257 + prev_tokens=previous_tokens,
258 + prefix_offset=prefix_offset,
259 + read_offset=read_offset,
260 + skip_special_tokens=False,
261 + spaces_between_special_tokens=True,
262 + )
263 +
264 + current_text = previous_text + delta_text
265 +
266 + delta_message = seed_oss_tool_parser.extract_tool_calls_streaming(
267 + previous_text,
268 + current_text,
269 + delta_text,
270 + previous_token_ids,
271 + current_token_ids,
272 + delta_token_ids,
273 + request=request,
274 + )
275 + if delta_message:
276 + yield delta_message
277 +
278 + previous_text = current_text
279 + previous_tokens = (previous_tokens +
280 + new_tokens if previous_tokens else new_tokens)
281 + prefix_offset = new_prefix_offset
282 + read_offset = new_read_offset
283 +
284 +
285 +@pytest.mark.parametrize(
286 + ids=[
287 + "tool_call_0_thinking_budget",
288 + "tool_call_512_thinkg_budget",
289 + "tool_call_unlimited_thinking_budget",
290 + ],
291 + argnames=["model_output", "expected_tool_calls", "expected_content"],
292 + argvalues=[
293 + ("""<seed:think>\n</seed:cot_budget_reflect>\n</seed:cot_budget_reflect>\n"""
294 + """The current thinking budget is 0, so I will directly start answering the question.\n</seed:think>\n"""
295 + """<seed:tool_call>\n<function=get_weather>\n"""
296 + """<parameter=location>Barcelona, Spain</parameter>\n</function>\n</seed:tool_call>""",
297 + [
298 + ToolCall(function=FunctionCall(
299 + name="get_weather",
300 + arguments=json.dumps({
301 + "location": "Barcelona, Spain",
302 + }, ),
303 + ),
304 + type='function')
305 + ],
306 + """<seed:think>\n</seed:cot_budget_reflect>\n</seed:cot_budget_reflect>\n"""
307 + """The current thinking budget is 0, so I will directly start answering the question.\n</seed:think>\n"""
308 + ),
309 + (
310 + """<seed:think>The user\'s current thinking budget is 512.</seed:cot_budget_reflect>\nLet me analyze the """
311 + """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """
312 + """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """
313 + """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """
314 + """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """
315 + """country). \n<seed:cot_budget_reflect>I have used 131 tokens, and there are 381 tokens remaining for use."""
316 + """</seed:cot_budget_reflect>\n Since the unit isn\'t specified, the function will default to Celsius, which """
317 + """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """
318 + """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """
319 + """user\'s input has a space, but the function might accept either; to be safe, using the standard format """
320 + """with a comma).\n<seed:cot_budget_reflect>I have used 257 tokens, and there are 255 tokens remaining for """
321 + """use.</seed:cot_budget_reflect>\n The unit parameter can be omitted since it\'s optional.</seed:think>\n"""
322 + """<seed:tool_call>\n<function=get_weather>\n<parameter=location>Barcelona, Spain</parameter>\n</function>"""
323 + """\n</seed:tool_call>""",
324 + [
325 + ToolCall(function=FunctionCall(
326 + name="get_weather",
327 + arguments=json.dumps({
328 + "location": "Barcelona, Spain",
329 + }, ),
330 + ),
331 + type='function')
332 + ],
333 + """<seed:think>The user\'s current thinking budget is 512.</seed:cot_budget_reflect>\nLet me analyze the """
334 + """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """
335 + """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """
336 + """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """
337 + """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """
338 + """country). \n<seed:cot_budget_reflect>I have used 131 tokens, and there are 381 tokens remaining for use."""
339 + """</seed:cot_budget_reflect>\n Since the unit isn\'t specified, the function will default to Celsius, which """
340 + """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """
341 + """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """
342 + """user\'s input has a space, but the function might accept either; to be safe, using the standard format """
343 + """with a comma).\n<seed:cot_budget_reflect>I have used 257 tokens, and there are 255 tokens remaining for """
344 + """use.</seed:cot_budget_reflect>\n The unit parameter can be omitted since it\'s optional.</seed:think>\n""",
345 + ),
346 + (
347 + """<seed:think>\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """
348 + """First, I need to remember the function I can use: get_weather. The function requires a """
349 + """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """
350 + """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """
351 + """let me check the function docstring again. Oh, the function says unit is optional, and """
352 + """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """
353 + """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """
354 + """The format is <seed:tool_call>\n<function=get_weather>\n<parameter=location>Barcelona, """
355 + """Spain</parameter>\n<parameter=unit>celsius</parameter>\n</function>\n</seed:tool_call>. """
356 + """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """
357 + """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """
358 + """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """
359 + """call should be as above. Then wait for the result to come back and tell the user the """
360 + """temperature in Celsius.</seed:think><seed:tool_call>\n<function=get_weather>\n<parameter=location>"""
361 + """Barcelona, Spain</parameter>\n<parameter=unit>celsius</parameter>\n</function>\n</seed:tool_call>""",
362 + [
363 + ToolCall(function=FunctionCall(
364 + name="get_weather",
365 + arguments=json.dumps(
366 + {
367 + "location": "Barcelona, Spain",
368 + "unit": "celsius",
369 + }, ),
370 + ),
371 + type='function')
372 + ],
373 + """<seed:think>\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """
374 + """First, I need to remember the function I can use: get_weather. The function requires a """
375 + """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """
376 + """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """
377 + """let me check the function docstring again. Oh, the function says unit is optional, and """
378 + """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """
379 + """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """
380 + """The format is <seed:tool_call>\n<function=get_weather>\n<parameter=location>Barcelona, """
381 + """Spain</parameter>\n<parameter=unit>celsius</parameter>\n</function>\n</seed:tool_call>. """
382 + """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """
383 + """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """
384 + """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """
385 + """call should be as above. Then wait for the result to come back and tell the user the """
386 + """temperature in Celsius.</seed:think>""",
387 + ),
388 + ],
389 +)
390 +def test_streaming_tool_calls(seed_oss_tool_parser, seed_oss_tokenizer,
391 + sample_tools, model_output, expected_tool_calls,
392 + expected_content):
393 + """Test incremental streaming behavior"""
394 + request = ChatCompletionRequest(model=MODEL,
395 + messages=[],
396 + tools=sample_tools)
397 +
398 + other_content = ''
399 + tool_states = {} # Track state per tool index
400 +
401 + for delta_message in stream_delta_message_generator(
402 + seed_oss_tool_parser, seed_oss_tokenizer, model_output, request):
403 + # role should never be streamed from tool parser
404 + assert not delta_message.role
405 +
406 + if delta_message.content:
407 + other_content += delta_message.content
408 +
409 + if delta_message.tool_calls:
410 + for tool_call in delta_message.tool_calls:
411 + idx = tool_call.index
412 +
413 + # Initialize state for new tool
414 + if idx not in tool_states:
415 + tool_states[idx] = {
416 + "id": None,
417 + "name": None,
418 + "arguments": "",
419 + "type": None
420 + }
421 +
422 + # First chunk should have id, name, and type
423 + if tool_call.id:
424 + tool_states[idx]["id"] = tool_call.id
425 +
426 + if tool_call.type:
427 + assert tool_call.type == "function"
428 + tool_states[idx]["type"] = tool_call.type
429 +
430 + if tool_call.function:
431 + if tool_call.function.name:
432 + # Should only be set once
433 + assert tool_states[idx]["name"] is None
434 + tool_states[idx]["name"] = tool_call.function.name
435 +
436 + if tool_call.function.arguments is not None:
437 + # Accumulate arguments incrementally
438 + tool_states[idx][
439 + "arguments"] += tool_call.function.arguments
440 +
441 + # Verify final content
442 + assert other_content == expected_content
443 +
444 + # Verify we got all expected tool calls
445 + assert len(tool_states) == len(expected_tool_calls)
446 +
447 + # Verify each tool call
448 + for idx, expected_tool in enumerate(expected_tool_calls):
449 + state = tool_states[idx]
450 + assert state["id"] is not None
451 + assert state["type"] == "function"
452 + assert state["name"] == expected_tool.function.name
453 +
454 + # Parse accumulated arguments
455 + arguments_str = state["arguments"]
456 + assert arguments_str is not None
457 + actual_args = json.loads(arguments_str)
458 + expected_args = json.loads(expected_tool.function.arguments)
459 + assert actual_args == expected_args
vllm/entrypoints/openai/tool_parsers/__init__.py
+2
@@ -18,6 +18,7 @@ from .mistral_tool_parser import MistralToolParser
18 from .phi4mini_tool_parser import Phi4MiniJsonToolParser
19 from .pythonic_tool_parser import PythonicToolParser
20 from .qwen3coder_tool_parser import Qwen3CoderToolParser
21 +from .seed_oss_tool_parser import SeedOssToolParser
22 from .step3_tool_parser import Step3ToolParser
23 from .xlam_tool_parser import xLAMToolParser
24
@@ -41,5 +42,6 @@ __all__ = [
42 "HunyuanA13BToolParser",
43 "Glm4MoeModelToolParser",
44 "Qwen3CoderToolParser",
45 + "SeedOssToolParser",
46 "Step3ToolParser",
47 ]
vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py new
+676
@@ -0,0 +1,676 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# Adapted from qwen3coder xml parser, All rights reserved.
4 +# ruff: noqa: E501
5 +
6 +import ast
7 +import json
8 +import uuid
9 +from collections.abc import Sequence
10 +from typing import Any, Optional, Union
11 +
12 +import regex as re
13 +
14 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
15 + ChatCompletionToolsParam,
16 + DeltaFunctionCall, DeltaMessage,
17 + DeltaToolCall,
18 + ExtractedToolCallInformation,
19 + FunctionCall, ToolCall)
20 +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
21 + ToolParser, ToolParserManager)
22 +from vllm.logger import init_logger
23 +from vllm.transformers_utils.tokenizer import AnyTokenizer
24 +
25 +logger = init_logger(__name__)
26 +
27 +
28 +@ToolParserManager.register_module("seed_oss")
29 +class SeedOssToolParser(ToolParser):
30 + TOOL_CALL_START = "<seed:tool_call>"
31 + TOOL_CALL_END = "</seed:tool_call>"
32 +
33 + def __init__(self, tokenizer: AnyTokenizer):
34 + super().__init__(tokenizer)
35 +
36 + # --- streaming state ---
37 + self._reset_streaming_state()
38 + self.prev_tool_call_arr: list[dict] = []
39 +
40 + self.tool_call_start_token: str = self.TOOL_CALL_START
41 + self.tool_call_end_token: str = self.TOOL_CALL_END
42 + # Sentinel tokens for streaming mode
43 + self.tool_call_prefix: str = "<function="
44 + self.function_end_token: str = "</function>"
45 + self.parameter_prefix: str = "<parameter="
46 + self.parameter_end_token: str = "</parameter>"
47 + self.think_start_token: str = "<seed:think>"
48 + self.think_end_token: str = "</seed:think>"
49 + self.is_tool_call_started: bool = False
50 + self.is_thinking_end: bool = False
51 + self.failed_count: int = 0
52 + self._reset_streaming_state()
53 +
54 + self.tool_call_start_token_id = self.vocab.get(
55 + self.tool_call_start_token)
56 + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
57 + self.think_end_token_id = self.vocab.get(self.think_end_token)
58 +
59 + if (self.tool_call_start_token_id is None
60 + or self.tool_call_end_token_id is None):
61 + raise RuntimeError(
62 + "Seed_Oss XML parser: tokenizer did not include "
63 + "<seed:tool_call> or its closing tag.")
64 +
65 + tool_start_re = re.escape(self.tool_call_start_token)
66 + tool_end_re = re.escape(self.tool_call_end_token)
67 +
68 + self.tool_call_complete_regex = re.compile(
69 + rf"{tool_start_re}(.*?){tool_end_re}", re.DOTALL)
70 + self.tool_call_regex = re.compile(
71 + rf"{tool_start_re}(.*?){tool_end_re}|{tool_start_re}(.*?)$",
72 + re.DOTALL)
73 +
74 + self.tool_call_function_regex = re.compile(
75 + r"<function=(.*?)</function>|<function=(.*)$", re.DOTALL)
76 + self.tool_call_parameter_regex = re.compile(
77 + r"<parameter=(.*?)</parameter>|<parameter=(.*?)$", re.DOTALL)
78 +
79 + logger.info("vLLM Seed-Oss XML tool parser loaded (%s).",
80 + self.__class__.__name__)
81 +
82 + def _generate_tool_call_id(self) -> str:
83 + """Generate a unique tool call ID."""
84 + return f"call_{uuid.uuid4().hex[:24]}"
85 +
86 + def _reset_streaming_state(self):
87 + """Reset all streaming state."""
88 + self.current_tool_index = 0
89 + self.is_tool_call_started = False
90 + self.header_sent = False
91 + self.current_tool_id = -1
92 + self.current_function_name = None
93 + self.current_param_name = None
94 + self.current_param_value = ""
95 + self.param_count = 0
96 + self.in_param = False
97 + self.in_function = False
98 + self.accumulated_text = ""
99 + self.json_started = False
100 + self.json_closed = False
101 +
102 + def _parse_xml_function_call(
103 + self, function_call_str: str,
104 + tools: Optional[list[ChatCompletionToolsParam]]
105 + ) -> Optional[ToolCall]:
106 +
107 + def get_arguments_config(func_name: str) -> dict:
108 + if tools is None:
109 + return {}
110 + for config in tools:
111 + if not hasattr(config, "type") or not (
112 + hasattr(config, "function")
113 + and hasattr(config.function, "name")):
114 + continue
115 + if (config.type == "function"
116 + and config.function.name == func_name):
117 + if not hasattr(config.function, "parameters"):
118 + return {}
119 + params = config.function.parameters
120 + if isinstance(params, dict) and "properties" in params:
121 + return params["properties"]
122 + elif isinstance(params, dict):
123 + return params
124 + else:
125 + return {}
126 + logger.warning("Tool '%s' is not defined in the tools list.",
127 + func_name)
128 + return {}
129 +
130 + def convert_param_value(param_value: str, param_name: str,
131 + param_config: dict, func_name: str) -> Any:
132 + # Handle null value for any type
133 + if param_value.lower() == "null":
134 + return None
135 +
136 + if param_name not in param_config:
137 + if param_config != {}:
138 + logger.warning(
139 + "Parsed parameter '%s' is not defined in "
140 + "the tool parameters for tool '%s', "
141 + "directly returning the string value.", param_name,
142 + func_name)
143 + return param_value
144 +
145 + if (isinstance(param_config[param_name], dict)
146 + and "type" in param_config[param_name]):
147 + param_type = str(
148 + param_config[param_name]["type"]).strip().lower()
149 + else:
150 + param_type = "string"
151 + if param_type in [
152 + "string", "str", "text", "varchar", "char", "enum"
153 + ]:
154 + return param_value
155 + elif (param_type.startswith("int") or param_type.startswith("uint")
156 + or param_type.startswith("long")
157 + or param_type.startswith("short")
158 + or param_type.startswith("unsigned")):
159 + try:
160 + param_value = int(param_value) # type: ignore
161 + except (ValueError, TypeError):
162 + logger.warning(
163 + "Parsed value '%s' of parameter '%s' is not an integer in tool "
164 + "'%s', degenerating to string.", param_value,
165 + param_name, func_name)
166 + return param_value
167 + elif param_type.startswith("num") or param_type.startswith(
168 + "float"):
169 + try:
170 + float_param_value = float(param_value)
171 + param_value = float_param_value if float_param_value - int(
172 + float_param_value) != 0 else int(
173 + float_param_value) # type: ignore
174 + except (ValueError, TypeError):
175 + logger.warning(
176 + "Parsed value '%s' of parameter '%s' is not a float in tool "
177 + "'%s', degenerating to string.", param_value,
178 + param_name, func_name)
179 + return param_value
180 + elif param_type in ["boolean", "bool", "binary"]:
181 + param_value = param_value.lower()
182 + if param_value not in ["true", "false"]:
183 + logger.warning(
184 + "Parsed value '%s' of parameter '%s' is not a boolean "
185 + "(`true` of `false`) in tool '%s', degenerating to false.",
186 + param_value, param_name, func_name)
187 + return param_value == "true"
188 + else:
189 + if param_type == "object" or param_type.startswith("dict"):
190 + try:
191 + param_value = json.loads(param_value)
192 + return param_value
193 + except (ValueError, TypeError, json.JSONDecodeError):
194 + logger.warning(
195 + "Parsed value '%s' of parameter '%s' is not a valid JSON "
196 + "object in tool '%s', will try other methods to parse it.",
197 + param_value, param_name, func_name)
198 + try:
199 + param_value = ast.literal_eval(param_value)
200 + except (ValueError, SyntaxError):
201 + logger.warning(
202 + "Parsed value '%s' of parameter '%s' cannot be converted via "
203 + "Python `ast.literal_eval()` in tool '%s', degenerating to string.",
204 + param_value, param_name, func_name)
205 + return param_value
206 +
207 + # Extract function name
208 + end_index = function_call_str.index(">")
209 + function_name = function_call_str[:end_index]
210 + param_config = get_arguments_config(function_name)
211 + parameters = function_call_str[end_index + 1:]
212 + param_dict = {}
213 + for match in self.tool_call_parameter_regex.findall(parameters):
214 + match_text = match[0] if match[0] else match[1]
215 + idx = match_text.index(">")
216 + param_name = match_text[:idx]
217 + param_value = str(match_text[idx + 1:])
218 + # Remove prefix and trailing \n
219 + if param_value.startswith("\n"):
220 + param_value = param_value[1:]
221 + if param_value.endswith("\n"):
222 + param_value = param_value[:-1]
223 +
224 + param_dict[param_name] = convert_param_value(
225 + param_value, param_name, param_config, function_name)
226 + return ToolCall(
227 + type="function",
228 + function=FunctionCall(name=function_name,
229 + arguments=json.dumps(param_dict,
230 + ensure_ascii=False)),
231 + )
232 +
233 + def _get_function_calls(self, model_output: str) -> list[str]:
234 + # Find all tool calls
235 + matched_ranges = self.tool_call_regex.findall(model_output)
236 + raw_tool_calls = [
237 + match[0] if match[0] else match[1] for match in matched_ranges
238 + ]
239 +
240 + # Back-off strategy if no tool_call tags found
241 + if len(raw_tool_calls) == 0:
242 + raw_tool_calls = [model_output]
243 +
244 + raw_function_calls = []
245 + for tool_call in raw_tool_calls:
246 + raw_function_calls.extend(
247 + self.tool_call_function_regex.findall(tool_call))
248 +
249 + function_calls = [
250 + match[0] if match[0] else match[1] for match in raw_function_calls
251 + ]
252 + return function_calls
253 +
254 + def extract_tool_calls(
255 + self,
256 + model_output: str,
257 + request: ChatCompletionRequest,
258 + ) -> ExtractedToolCallInformation:
259 + # Quick check to avoid unnecessary processing
260 + if self.tool_call_prefix not in model_output:
261 + return ExtractedToolCallInformation(tools_called=False,
262 + tool_calls=[],
263 + content=model_output)
264 +
265 + # Check if both think start and end tokens are present
266 + if (self.think_start_token in model_output
267 + and self.think_end_token in model_output):
268 + # Find the position of think end token
269 + think_end_index = model_output.find(self.think_end_token) + len(
270 + self.think_end_token)
271 + # Extract content after think end token
272 + result_content = model_output[think_end_index:]
273 + thinking_content = model_output[:think_end_index]
274 +
275 + try:
276 + function_calls = self._get_function_calls(result_content)
277 + if len(function_calls) == 0:
278 + return ExtractedToolCallInformation(tools_called=False,
279 + tool_calls=[],
280 + content=model_output)
281 +
282 + tool_calls = [
283 + self._parse_xml_function_call(function_call_str, request.tools)
284 + for function_call_str in function_calls
285 + ]
286 +
287 + # Populate prev_tool_call_arr for serving layer to set finish_reason
288 + self.prev_tool_call_arr.clear() # Clear previous calls
289 + for tool_call in tool_calls:
290 + if tool_call:
291 + self.prev_tool_call_arr.append({
292 + "name":
293 + tool_call.function.name,
294 + "arguments":
295 + tool_call.function.arguments,
296 + })
297 +
298 + # Extract content before tool calls
299 + tool_call_start_index = result_content.find(
300 + self.tool_call_start_token)
301 + tool_call_start_index = (
302 + tool_call_start_index if tool_call_start_index >= 0 else
303 + result_content.find(self.tool_call_prefix))
304 + content = thinking_content + result_content[:tool_call_start_index]
305 +
306 + return ExtractedToolCallInformation(
307 + tools_called=(len(tool_calls) > 0),
308 + tool_calls=tool_calls,
309 + content=content if content else None,
310 + )
311 +
312 + except Exception:
313 + logger.exception("Error in extracting tool call from response.")
314 + return ExtractedToolCallInformation(tools_called=False,
315 + tool_calls=[],
316 + content=model_output)
317 +
318 + def extract_tool_calls_streaming(
319 + self,
320 + previous_text: str,
321 + current_text: str,
322 + delta_text: str,
323 + previous_token_ids: Sequence[int],
324 + current_token_ids: Sequence[int],
325 + delta_token_ids: Sequence[int],
326 + request: ChatCompletionRequest,
327 + ) -> Union[DeltaMessage, None]:
328 + # If no delta text, return None unless
329 + # it's an EOS token after tool calls
330 + if not delta_text:
331 + # Check if this is an EOS token after all tool calls are complete
332 + # We check for tool calls in the text even if is_tool_call_started
333 + # is False because it might have been reset after processing all tools
334 + if (delta_token_ids
335 + and self.tool_call_end_token_id not in delta_token_ids):
336 + # Count complete tool calls
337 + complete_calls = len(
338 + self.tool_call_complete_regex.findall(current_text))
339 +
340 + # If we have completed tool calls and populated prev_tool_call_arr
341 + if complete_calls > 0 and len(self.prev_tool_call_arr) > 0:
342 + # Check if all tool calls are closed
343 + open_calls = current_text.count(
344 + self.tool_call_start_token) - current_text.count(
345 + self.tool_call_end_token)
346 + if open_calls == 0:
347 + # Return empty delta message to allow finish_reason processing
348 + return DeltaMessage(content="")
349 + elif not self.is_tool_call_started and current_text:
350 + # This is a regular content response that's now complete
351 + return DeltaMessage(content="")
352 + return None
353 +
354 + # Check if this is the first call (reset state if needed)
355 + if not previous_text:
356 + self._reset_streaming_state()
357 +
358 + # Update accumulated text
359 + self.accumulated_text = current_text
360 +
361 + # Check if we need to advance to next tool
362 + if self.json_closed and not self.in_function:
363 + # Check if this tool call has ended
364 + tool_ends = current_text.count(self.tool_call_end_token)
365 + if tool_ends > self.current_tool_index:
366 + # This tool has ended, advance to next
367 + self.current_tool_index += 1
368 + self.header_sent = False
369 + self.param_count = 0
370 + self.json_started = False
371 + self.json_closed = False
372 +
373 + # Check if there are more tool calls
374 + if self.current_tool_index >= current_text.count(
375 + self.tool_call_start_token):
376 + # No more tool calls
377 + self.is_tool_call_started = False
378 + # Continue processing next tool
379 + return None
380 +
381 + # Check if end thinking
382 + if (not self.is_thinking_end
383 + and (self.think_end_token_id in delta_token_ids
384 + or self.think_end_token in delta_text)):
385 + self.is_thinking_end = True
386 +
387 + # If thinking hasn't ended yet, don't process any tool calls
388 + if not self.is_thinking_end:
389 + return DeltaMessage(content=delta_text)
390 +
391 + # Handle normal content before tool calls
392 + if not self.is_tool_call_started:
393 + # Check if tool call is starting
394 + if (self.tool_call_start_token_id in delta_token_ids
395 + or self.tool_call_start_token in delta_text):
396 + self.is_tool_call_started = True
397 + # Return any content before the tool call
398 + if self.tool_call_start_token in delta_text:
399 + content_before = delta_text[:delta_text.index(
400 + self.tool_call_start_token)]
401 + if content_before:
402 + return DeltaMessage(content=content_before)
403 + return None
404 + else:
405 + # Check if we're between tool calls - skip whitespace
406 + if (current_text.rstrip().endswith(self.tool_call_end_token)
407 + and delta_text.strip() == ""):
408 + # We just ended a tool call, skip whitespace
409 + return None
410 + # Normal content, no tool call
411 + return DeltaMessage(content=delta_text)
412 +
413 + # Check if we're between tool calls (waiting for next one)
414 + # Count tool calls we've seen vs processed
415 + tool_starts_count = current_text.count(self.tool_call_start_token)
416 + if self.current_tool_index >= tool_starts_count:
417 + # We're past all tool calls, shouldn't be here
418 + return None
419 +
420 + # We're in a tool call, find the current tool call portion
421 + # Need to find the correct tool call based on current_tool_index
422 + # Only process tool calls after think_end_token
423 + think_end_index = current_text.find(self.think_end_token) + len(
424 + self.think_end_token
425 + ) if self.think_end_token in current_text else 0
426 + tool_starts: list[int] = []
427 + idx = think_end_index
428 + while True:
429 + idx = current_text.find(self.tool_call_start_token, idx)
430 + if idx == -1:
431 + break
432 + tool_starts.append(idx)
433 + idx += len(self.tool_call_start_token)
434 +
435 + if self.current_tool_index >= len(tool_starts):
436 + # No more tool calls to process yet
437 + return None
438 +
439 + tool_start_idx = tool_starts[self.current_tool_index]
440 + # Find where this tool call ends (or current position if not ended yet)
441 + tool_end_idx = current_text.find(self.tool_call_end_token,
442 + tool_start_idx)
443 + if tool_end_idx == -1:
444 + tool_text = current_text[tool_start_idx:]
445 + else:
446 + tool_text = current_text[tool_start_idx:tool_end_idx +
447 + len(self.tool_call_end_token)]
448 +
449 + # Looking for function header
450 + if not self.header_sent:
451 + if self.tool_call_prefix in tool_text:
452 + func_start = tool_text.find(self.tool_call_prefix) + len(
453 + self.tool_call_prefix)
454 + func_end = tool_text.find(">", func_start)
455 +
456 + if func_end != -1:
457 + # Found complete function name
458 + self.current_function_name = tool_text[func_start:func_end]
459 + self.current_tool_id = self._generate_tool_call_id(
460 + ) # type: ignore
461 + self.header_sent = True
462 + self.in_function = True
463 +
464 + # IMPORTANT: Add to prev_tool_call_arr immediately when we detect a tool call
465 + # This ensures finish_reason="tool_calls" even if parsing isn't complete
466 + already_added = any(
467 + tool.get("name") == self.current_function_name
468 + for tool in self.prev_tool_call_arr)
469 + if not already_added:
470 + self.prev_tool_call_arr.append({
471 + "name": self.current_function_name,
472 + "arguments":
473 + "{}", # Placeholder, will be updated later
474 + })
475 +
476 + # Send header with function info
477 + return DeltaMessage(tool_calls=[
478 + DeltaToolCall(
479 + index=self.current_tool_index,
480 + id=self.current_tool_id,
481 + function=DeltaFunctionCall(
482 + name=self.current_function_name, arguments=""),
483 + type="function",
484 + )
485 + ])
486 + return None
487 +
488 + # We've sent header, now handle function body
489 + if self.in_function:
490 + # Send opening brace if not sent yet
491 + if (not self.json_started
492 + and self.parameter_prefix not in delta_text):
493 + self.json_started = True
494 + return DeltaMessage(tool_calls=[
495 + DeltaToolCall(
496 + index=self.current_tool_index,
497 + function=DeltaFunctionCall(arguments="{"),
498 + )
499 + ])
500 +
501 + # Make sure json_started is set if we're processing parameters
502 + if not self.json_started:
503 + self.json_started = True
504 +
505 + # Check for function end in accumulated text
506 + if not self.json_closed and self.function_end_token in tool_text:
507 + # Close JSON
508 + self.json_closed = True
509 +
510 + # Extract the complete tool call to update prev_tool_call_arr with final arguments
511 + # Find the function content
512 + func_start = tool_text.find(self.tool_call_prefix) + len(
513 + self.tool_call_prefix)
514 + func_content_end = tool_text.find(self.function_end_token,
515 + func_start)
516 + if func_content_end != -1:
517 + func_content = tool_text[func_start:func_content_end]
518 + # Parse to get the complete arguments
519 + try:
520 + parsed_tool = self._parse_xml_function_call(
521 + func_content, request.tools if request else None)
522 + if parsed_tool:
523 + # Update existing entry in prev_tool_call_arr with complete arguments
524 + for i, tool in enumerate(self.prev_tool_call_arr):
525 + if tool.get(
526 + "name") == parsed_tool.function.name:
527 + self.prev_tool_call_arr[i]["arguments"] = (
528 + parsed_tool.function.arguments)
529 + break
530 + except Exception:
531 + logger.warning(
532 + "Failed to parse tool arguments during streaming.",
533 + exc_info=True)
534 +
535 + result = DeltaMessage(tool_calls=[
536 + DeltaToolCall(
537 + index=self.current_tool_index,
538 + function=DeltaFunctionCall(arguments="}"),
539 + )
540 + ])
541 +
542 + # Reset state for next tool
543 + self.in_function = False
544 + self.json_closed = True
545 +
546 + return result
547 +
548 + # Look for parameters
549 + # Count how many complete parameters we have processed
550 + complete_params = tool_text.count(self.parameter_end_token)
551 +
552 + # Check if we should start a new parameter
553 + if not self.in_param and self.param_count < complete_params:
554 + # Find the unprocessed parameter
555 + # Count parameter starts
556 + param_starts = []
557 + idx = 0
558 + while True:
559 + idx = tool_text.find(self.parameter_prefix, idx)
560 + if idx == -1:
561 + break
562 + param_starts.append(idx)
563 + idx += len(self.parameter_prefix)
564 +
565 + if len(param_starts) > self.param_count:
566 + # Process the next parameter
567 + param_idx = param_starts[self.param_count]
568 + param_start = param_idx + len(self.parameter_prefix)
569 + remaining = tool_text[param_start:]
570 +
571 + if ">" in remaining:
572 + # We have the complete parameter name
573 + name_end = remaining.find(">")
574 + self.current_param_name = remaining[:name_end]
575 +
576 + # Find the parameter value
577 + value_start = param_start + name_end + 1
578 + value_text = tool_text[value_start:]
579 + if value_text.startswith("\n"):
580 + value_text = value_text[1:]
581 +
582 + # Find where this parameter ends
583 + param_end_idx = value_text.find(
584 + self.parameter_end_token)
585 + if param_end_idx != -1:
586 + # Complete parameter found
587 + param_value = value_text[:param_end_idx]
588 + if param_value.endswith("\n"):
589 + param_value = param_value[:-1]
590 +
591 + # Build complete JSON fragment for this parameter
592 + if self.param_count == 0:
593 + json_fragment = (
594 + '"' + self.current_param_name + '": "' +
595 + json.dumps(param_value)[1:-1] + '"')
596 + else:
597 + json_fragment = (
598 + ', "' + self.current_param_name + '": "' +
599 + json.dumps(param_value)[1:-1] + '"')
600 +
601 + self.param_count += 1
602 +
603 + return DeltaMessage(tool_calls=[
604 + DeltaToolCall(
605 + index=self.current_tool_index,
606 + function=DeltaFunctionCall(
607 + arguments=json_fragment),
608 + )
609 + ])
610 +
611 + # Continue parameter value
612 + if self.in_param:
613 + if self.parameter_end_token in delta_text:
614 + # End of parameter
615 + end_idx = delta_text.find(self.parameter_end_token)
616 + value_chunk = delta_text[:end_idx]
617 +
618 + # Skip past > if at start
619 + if not self.current_param_value and ">" in value_chunk:
620 + gt_idx = value_chunk.find(">")
621 + value_chunk = value_chunk[gt_idx + 1:]
622 +
623 + if not self.current_param_value and value_chunk.startswith(
624 + "\n"):
625 + value_chunk = value_chunk[1:]
626 +
627 + # Calculate incremental JSON
628 + full_value = self.current_param_value + value_chunk
629 + prev_escaped = (json.dumps(self.current_param_value)[1:-1]
630 + if self.current_param_value else "")
631 + full_escaped = json.dumps(full_value)[1:-1]
632 + delta_escaped = full_escaped[len(prev_escaped):]
633 +
634 + self.in_param = False
635 + self.current_param_value = ""
636 +
637 + return DeltaMessage(tool_calls=[
638 + DeltaToolCall(
639 + index=self.current_tool_index,
640 + function=DeltaFunctionCall(
641 + arguments=delta_escaped + '"'),
642 + )
643 + ])
644 + else:
645 + # Continue accumulating value
646 + value_chunk = delta_text
647 +
648 + # Handle first chunk after param name
649 + if not self.current_param_value and ">" in value_chunk:
650 + gt_idx = value_chunk.find(">")
651 + value_chunk = value_chunk[gt_idx + 1:]
652 +
653 + if not self.current_param_value and value_chunk.startswith(
654 + "\n"):
655 + value_chunk = value_chunk[1:]
656 +
657 + if value_chunk:
658 + # Stream the escaped delta
659 + prev_escaped = (json.dumps(
660 + self.current_param_value)[1:-1]
661 + if self.current_param_value else "")
662 + self.current_param_value += value_chunk
663 + full_escaped = json.dumps(
664 + self.current_param_value)[1:-1]
665 + delta_escaped = full_escaped[len(prev_escaped):]
666 +
667 + if delta_escaped:
668 + return DeltaMessage(tool_calls=[
669 + DeltaToolCall(
670 + index=self.current_tool_index,
671 + function=DeltaFunctionCall(
672 + arguments=delta_escaped),
673 + )
674 + ])
675 +
676 + return None
vllm/model_executor/models/registry.py
+1
@@ -130,6 +130,7 @@ _TEXT_GENERATION_MODELS = {
130 "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),
131 "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"),
132 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
133 + "SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
134 "Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
135 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
136 "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
vllm/model_executor/models/seed_oss.py new
+487
@@ -0,0 +1,487 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# Copyright 2025 The Seed team.
5 +# Copyright 2023 The vLLM team.
6 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7 +#
8 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only SeedOss model compatible with HuggingFace weights."""
25 +from collections.abc import Iterable
26 +from typing import Optional, Union
27 +
28 +import torch
29 +from torch import nn
30 +from transformers import PretrainedConfig as SeedOssConfig
31 +
32 +from vllm.attention import Attention, AttentionType
33 +from vllm.compilation.decorators import support_torch_compile
34 +from vllm.config import CacheConfig, VllmConfig
35 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
36 +from vllm.logger import init_logger
37 +from vllm.model_executor.layers.activation import SiluAndMul
38 +from vllm.model_executor.layers.layernorm import RMSNorm
39 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
40 + QKVParallelLinear,
41 + RowParallelLinear)
42 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 +from vllm.model_executor.layers.quantization import QuantizationConfig
44 +from vllm.model_executor.layers.rotary_embedding import get_rope
45 +from vllm.model_executor.layers.vocab_parallel_embedding import (
46 + ParallelLMHead, VocabParallelEmbedding)
47 +from vllm.model_executor.model_loader.weight_utils import (
48 + default_weight_loader, maybe_remap_kv_scale_name)
49 +from vllm.model_executor.sampling_metadata import SamplingMetadata
50 +from vllm.sequence import IntermediateTensors
51 +
52 +from .interfaces import SupportsLoRA, SupportsPP
53 +from .utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter,
54 + make_empty_intermediate_tensors_factory, make_layers,
55 + maybe_prefix)
56 +
57 +logger = init_logger(__name__)
58 +
59 +
60 +class SeedOssMLP(nn.Module):
61 +
62 + def __init__(
63 + self,
64 + hidden_size: int,
65 + intermediate_size: int,
66 + hidden_act: str,
67 + quant_config: Optional[QuantizationConfig] = None,
68 + prefix: str = "",
69 + ) -> None:
70 + super().__init__()
71 + self.gate_up_proj = MergedColumnParallelLinear(
72 + hidden_size,
73 + [intermediate_size] * 2,
74 + bias=False,
75 + quant_config=quant_config,
76 + prefix=f"{prefix}.gate_up_proj",
77 + )
78 + self.down_proj = RowParallelLinear(
79 + intermediate_size,
80 + hidden_size,
81 + bias=False,
82 + quant_config=quant_config,
83 + prefix=f"{prefix}.down_proj",
84 + )
85 + if hidden_act != "silu":
86 + raise ValueError(f"Unsupported activation: {hidden_act}. "
87 + "Only silu is supported for now.")
88 + self.act_fn = SiluAndMul()
89 +
90 + def forward(self, x):
91 + gate_up, _ = self.gate_up_proj(x)
92 + x = self.act_fn(gate_up)
93 + x, _ = self.down_proj(x)
94 + return x
95 +
96 +
97 +class SeedOssAttention(nn.Module):
98 +
99 + def __init__(
100 + self,
101 + hidden_size: int,
102 + num_heads: int,
103 + num_kv_heads: int,
104 + head_dim: int,
105 + max_position: int = 4096 * 32,
106 + rope_theta: float = 10000,
107 + cache_config: Optional[CacheConfig] = None,
108 + quant_config: Optional[QuantizationConfig] = None,
109 + rope_scaling: Optional[tuple] = None,
110 + prefix: str = "",
111 + attn_type: str = AttentionType.DECODER,
112 + ) -> None:
113 + super().__init__()
114 + self.hidden_size = hidden_size
115 + tp_size = get_tensor_model_parallel_world_size()
116 + self.total_num_heads = num_heads
117 + assert self.total_num_heads % tp_size == 0
118 + self.num_heads = self.total_num_heads // tp_size
119 + self.total_num_kv_heads = num_kv_heads
120 + self.head_dim = head_dim
121 + if self.total_num_kv_heads >= tp_size:
122 + # Number of KV heads is greater than TP size, so we partition
123 + # the KV heads across multiple tensor parallel GPUs.
124 + assert self.total_num_kv_heads % tp_size == 0
125 + else:
126 + # Number of KV heads is less than TP size, so we replicate
127 + # the KV heads across multiple tensor parallel GPUs.
128 + assert tp_size % self.total_num_kv_heads == 0
129 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
130 + self.q_size = self.num_heads * self.head_dim
131 + self.kv_size = self.num_kv_heads * self.head_dim
132 + self.scaling = self.head_dim**-0.5
133 + self.rope_theta = rope_theta
134 +
135 + self.qkv_proj = QKVParallelLinear(
136 + hidden_size,
137 + self.head_dim,
138 + self.total_num_heads,
139 + self.total_num_kv_heads,
140 + bias=True,
141 + quant_config=quant_config,
142 + prefix=f"{prefix}.qkv_proj",
143 + )
144 + self.o_proj = RowParallelLinear(
145 + self.total_num_heads * self.head_dim,
146 + hidden_size,
147 + bias=False,
148 + quant_config=quant_config,
149 + prefix=f"{prefix}.o_proj",
150 + )
151 +
152 + self.rotary_emb = get_rope(
153 + self.head_dim,
154 + rotary_dim=self.head_dim,
155 + max_position=max_position,
156 + base=self.rope_theta,
157 + rope_scaling=rope_scaling,
158 + )
159 + self.attn = Attention(
160 + self.num_heads,
161 + self.head_dim,
162 + self.scaling,
163 + num_kv_heads=self.num_kv_heads,
164 + cache_config=cache_config,
165 + quant_config=quant_config,
166 + attn_type=attn_type,
167 + prefix=f"{prefix}.attn",
168 + )
169 +
170 + def forward(
171 + self,
172 + positions: torch.Tensor,
173 + hidden_states: torch.Tensor,
174 + ) -> torch.Tensor:
175 + qkv, _ = self.qkv_proj(hidden_states)
176 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
177 + q, k = self.rotary_emb(positions, q, k)
178 + attn_output = self.attn(q, k, v)
179 + output, _ = self.o_proj(attn_output)
180 + return output
181 +
182 +
183 +class SeedOssDecoderLayer(nn.Module):
184 +
185 + def __init__(
186 + self,
187 + config: SeedOssConfig,
188 + cache_config: Optional[CacheConfig] = None,
189 + quant_config: Optional[QuantizationConfig] = None,
190 + prefix: str = "",
191 + ) -> None:
192 + super().__init__()
193 + self.hidden_size = config.hidden_size
194 + # Requires transformers > 4.32.0
195 + rope_theta = getattr(config, "rope_theta", 1000000)
196 + rope_scaling = getattr(config, "rope_scaling", None)
197 +
198 + # By default, SeedOss uses causal attention as it is a
199 + # decoder-only model.
200 + # You can override the HF config with `is_causal=False` to enable
201 + # bidirectional attention, which is used in some embedding models
202 + if getattr(config, "is_causal", True):
203 + attn_type = AttentionType.DECODER
204 + else:
205 + attn_type = AttentionType.ENCODER_ONLY
206 +
207 + self.self_attn = SeedOssAttention(
208 + hidden_size=self.hidden_size,
209 + num_heads=config.num_attention_heads,
210 + max_position=config.max_position_embeddings,
211 + num_kv_heads=config.num_key_value_heads,
212 + head_dim=config.head_dim,
213 + rope_theta=rope_theta,
214 + cache_config=cache_config,
215 + quant_config=quant_config,
216 + rope_scaling=rope_scaling,
217 + prefix=f"{prefix}.self_attn",
218 + attn_type=attn_type,
219 + )
220 + self.mlp = SeedOssMLP(
221 + hidden_size=self.hidden_size,
222 + intermediate_size=config.intermediate_size,
223 + hidden_act=config.hidden_act,
224 + quant_config=quant_config,
225 + prefix=f"{prefix}.mlp",
226 + )
227 + self.input_layernorm = RMSNorm(config.hidden_size,
228 + eps=config.rms_norm_eps)
229 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
230 + eps=config.rms_norm_eps)
231 +
232 + def forward(
233 + self,
234 + positions: torch.Tensor,
235 + hidden_states: torch.Tensor,
236 + residual: Optional[torch.Tensor],
237 + ) -> tuple[torch.Tensor, torch.Tensor]:
238 + # Self Attention
239 + if residual is None:
240 + residual = hidden_states
241 + hidden_states = self.input_layernorm(hidden_states)
242 + else:
243 + hidden_states, residual = self.input_layernorm(
244 + hidden_states, residual)
245 + hidden_states = self.self_attn(
246 + positions=positions,
247 + hidden_states=hidden_states,
248 + )
249 +
250 + # Fully Connected
251 + hidden_states, residual = self.post_attention_layernorm(
252 + hidden_states, residual)
253 + hidden_states = self.mlp(hidden_states)
254 + return hidden_states, residual
255 +
256 +
257 +@support_torch_compile(
258 + dynamic_arg_dims={
259 + "input_ids": 0,
260 + "positions": -1,
261 + "intermediate_tensors": 0,
262 + "inputs_embeds": 0,
263 + })
264 +class SeedOssModel(nn.Module):
265 +
266 + def __init__(self,
267 + *,
268 + vllm_config: VllmConfig,
269 + prefix: str = "",
270 + decoder_layer_type: type[nn.Module] = SeedOssDecoderLayer):
271 + super().__init__()
272 +
273 + config = vllm_config.model_config.hf_config
274 + cache_config = vllm_config.cache_config
275 + quant_config = vllm_config.quant_config
276 +
277 + # TODO (@robertgshaw2): see if this can be moved out
278 + if (cache_config.sliding_window is not None
279 + and hasattr(config, "max_window_layers")):
280 + assert config.max_window_layers == config.num_hidden_layers, (
281 + "Sliding window for some but all layers is not supported. "
282 + "This model uses sliding window but `max_window_layers` = {} "
283 + "is less than `num_hidden_layers` = {}. Please open an issue "
284 + "to discuss this feature.".format(
285 + config.max_window_layers,
286 + config.num_hidden_layers,
287 + ))
288 +
289 + self.config = config
290 + self.quant_config = quant_config
291 + self.vocab_size = config.vocab_size
292 +
293 + if get_pp_group().is_first_rank or (config.tie_word_embeddings
294 + and get_pp_group().is_last_rank):
295 + self.embed_tokens = VocabParallelEmbedding(
296 + config.vocab_size,
297 + config.hidden_size,
298 + quant_config=quant_config,
299 + prefix=f"{prefix}.embed_tokens",
300 + )
301 + else:
302 + self.embed_tokens = PPMissingLayer()
303 +
304 + # Use the provided decoder layer type or default to SeedDecoderLayer
305 + decoder_layer_type = decoder_layer_type or SeedOssDecoderLayer
306 + self.start_layer, self.end_layer, self.layers = make_layers(
307 + config.num_hidden_layers,
308 + lambda prefix: decoder_layer_type(config=config,
309 + cache_config=cache_config,
310 + quant_config=quant_config,
311 + prefix=prefix),
312 + prefix=f"{prefix}.layers",
313 + )
314 +
315 + self.make_empty_intermediate_tensors = (
316 + make_empty_intermediate_tensors_factory(
317 + ["hidden_states", "residual"], config.hidden_size))
318 + if get_pp_group().is_last_rank:
319 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
320 + else:
321 + self.norm = PPMissingLayer()
322 +
323 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
324 + return self.embed_tokens(input_ids)
325 +
326 + def forward(
327 + self,
328 + input_ids: torch.Tensor,
329 + positions: torch.Tensor,
330 + intermediate_tensors: Optional[IntermediateTensors] = None,
331 + inputs_embeds: Optional[torch.Tensor] = None,
332 + ) -> Union[torch.Tensor, IntermediateTensors]:
333 + if get_pp_group().is_first_rank:
334 + if inputs_embeds is not None:
335 + hidden_states = inputs_embeds
336 + else:
337 + hidden_states = self.get_input_embeddings(input_ids)
338 + residual = None
339 + else:
340 + assert intermediate_tensors is not None
341 + hidden_states = intermediate_tensors["hidden_states"]
342 + residual = intermediate_tensors["residual"]
343 + for layer in self.layers[self.start_layer:self.end_layer]:
344 + hidden_states, residual = layer(
345 + positions,
346 + hidden_states,
347 + residual,
348 + )
349 + if not get_pp_group().is_last_rank:
350 + return IntermediateTensors({
351 + "hidden_states": hidden_states,
352 + "residual": residual
353 + })
354 + hidden_states, _ = self.norm(hidden_states, residual)
355 + return hidden_states
356 +
357 + def load_weights(self, weights: Iterable[tuple[str,
358 + torch.Tensor]]) -> set[str]:
359 + stacked_params_mapping = [
360 + # (param_name, shard_name, shard_id)
361 + ("qkv_proj", "q_proj", "q"),
362 + ("qkv_proj", "k_proj", "k"),
363 + ("qkv_proj", "v_proj", "v"),
364 + ("gate_up_proj", "gate_proj", 0),
365 + ("gate_up_proj", "up_proj", 1),
366 + ]
367 + params_dict = dict(self.named_parameters(remove_duplicate=False))
368 + loaded_params: set[str] = set()
369 + for name, loaded_weight in weights:
370 + if "rotary_emb.inv_freq" in name:
371 + continue
372 + if (self.quant_config is not None and
373 + (scale_name := self.quant_config.get_cache_scale(name))):
374 + # Loading kv cache quantization scales
375 + param = params_dict[scale_name]
376 + weight_loader = getattr(param, "weight_loader",
377 + default_weight_loader)
378 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
379 + loaded_weight[0])
380 + weight_loader(param, loaded_weight)
381 + loaded_params.add(scale_name)
382 + continue
383 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
384 + if weight_name not in name:
385 + continue
386 + name = name.replace(weight_name, param_name)
387 + # Skip loading extra bias for GPTQ models.
388 + if name.endswith(".bias") and name not in params_dict:
389 + continue
390 + if is_pp_missing_parameter(name, self):
391 + continue
392 + param = params_dict[name]
393 + weight_loader = param.weight_loader
394 + weight_loader(param, loaded_weight, shard_id)
395 + break
396 + else:
397 + # Skip loading extra bias for GPTQ models.
398 + if name.endswith(".bias") and name not in params_dict:
399 + continue
400 + # Remapping the name of FP8 kv-scale.
401 + name = maybe_remap_kv_scale_name(name, params_dict)
402 + if name is None:
403 + continue
404 + if is_pp_missing_parameter(name, self):
405 + continue
406 + param = params_dict[name]
407 + weight_loader = getattr(param, "weight_loader",
408 + default_weight_loader)
409 + weight_loader(param, loaded_weight)
410 + loaded_params.add(name)
411 + return loaded_params
412 +
413 +
414 +class SeedOssForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
415 + packed_modules_mapping = {
416 + "qkv_proj": [
417 + "q_proj",
418 + "k_proj",
419 + "v_proj",
420 + ],
421 + "gate_up_proj": [
422 + "gate_proj",
423 + "up_proj",
424 + ],
425 + }
426 +
427 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
428 + super().__init__()
429 + config = vllm_config.model_config.hf_config
430 + quant_config = vllm_config.quant_config
431 + lora_config = vllm_config.lora_config
432 +
433 + self.config = config
434 + self.lora_config = lora_config
435 +
436 + self.quant_config = quant_config
437 + self.model = SeedOssModel(vllm_config=vllm_config,
438 + prefix=maybe_prefix(prefix, "model"))
439 +
440 + if get_pp_group().is_last_rank:
441 + if config.tie_word_embeddings:
442 + self.lm_head = self.model.embed_tokens
443 + else:
444 + self.lm_head = ParallelLMHead(config.vocab_size,
445 + config.hidden_size,
446 + quant_config=quant_config,
447 + prefix=maybe_prefix(
448 + prefix, "lm_head"))
449 + else:
450 + self.lm_head = PPMissingLayer()
451 +
452 + self.logits_processor = LogitsProcessor(config.vocab_size)
453 +
454 + self.make_empty_intermediate_tensors = (
455 + self.model.make_empty_intermediate_tensors)
456 +
457 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
458 + return self.model.get_input_embeddings(input_ids)
459 +
460 + def forward(
461 + self,
462 + input_ids: torch.Tensor,
463 + positions: torch.Tensor,
464 + intermediate_tensors: Optional[IntermediateTensors] = None,
465 + inputs_embeds: Optional[torch.Tensor] = None,
466 + ) -> Union[torch.Tensor, IntermediateTensors]:
467 + hidden_states = self.model(input_ids, positions, intermediate_tensors,
468 + inputs_embeds)
469 + return hidden_states
470 +
471 + def compute_logits(
472 + self,
473 + hidden_states: torch.Tensor,
474 + sampling_metadata: SamplingMetadata,
475 + ) -> Optional[torch.Tensor]:
476 + logits = self.logits_processor(self.lm_head, hidden_states,
477 + sampling_metadata)
478 + return logits
479 +
480 + def load_weights(self, weights: Iterable[tuple[str,
481 + torch.Tensor]]) -> set[str]:
482 + loader = AutoWeightsLoader(
483 + self,
484 + skip_prefixes=(["lm_head."]
485 + if self.config.tie_word_embeddings else None),
486 + )
487 + return loader.load_weights(weights)