@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5a7fb3ab9e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 01:00 UTC

[Model] Add ToolParser and MoE Config for Hunyuan A13B (#20820)

Signed-off-by: Asher Zhang <asherszhang@tencent.com>

Asher committed Jul 17, 2025 at 17:10 UTC 5a7fb3ab9ee4bd8174a5100595326f6f40024cdc
17 files changed +1712 -4
benchmarks/kernels/benchmark_moe.py
+5
@@ -586,6 +586,11 @@ def main(args: argparse.Namespace):
586 topk = config.num_experts_per_tok
587 intermediate_size = config.moe_intermediate_size
588 shard_intermediate_size = 2 * intermediate_size // args.tp_size
589 + elif config.architectures[0] in ("HunYuanMoEV1ForCausalLM"):
590 + E = config.num_experts
591 + topk = config.moe_topk[0]
592 + intermediate_size = config.moe_intermediate_size[0]
593 + shard_intermediate_size = 2 * intermediate_size // args.tp_size
594 else:
595 # Support for llama4
596 config = config.get_text_config()
docs/features/reasoning_outputs.md
+1
@@ -14,6 +14,7 @@ vLLM currently supports the following reasoning models:
14 | [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `guided_json`, `guided_regex` | ✅ |
15 | [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ |
16 | [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `guided_json`, `guided_regex` | ✅ |
17 +| [Hunyuan A13B series](https://huggingface.co/collections/tencent/hunyuan-a13b-685ec38e5b46321e3ea7c4be) | `hunyuan_a13b` | `guided_json`, `guided_regex` | ✅ |
18
19 !!! note
20 IBM Granite 3.2 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.
docs/features/tool_calling.md
+10
@@ -288,6 +288,16 @@ Supported models:
288
289 Flags: `--tool-call-parser kimi_k2`
290
291 +### Hunyuan Models (`hunyuan_a13b`)
292 +
293 +Supported models:
294 +
295 +* `tencent/Hunyuan-A13B-Instruct` (chat template already included huggingface model file.)
296 +
297 +Flags:
298 +* For non-reasoning: `--tool-call-parser hunyuan_a13b`
299 +* For reasoning: `--tool-call-parser hunyuan_a13b --reasoning-parser hunyuan_a13b --enable_reasoning`
300 +
301 ### Models with Pythonic Tool Calls (`pythonic`)
302
303 A growing number of models output a python list to represent tool calls instead of using JSON. This has the advantage of inherently supporting parallel tool calls and removing ambiguity around the JSON schema required for tool calls. The `pythonic` tool parser can support such models.
examples/tool_chat_template_hunyuan_a13b.jinja new
+113
@@ -0,0 +1,113 @@
1 +{% set loop_messages = messages %}
2 +{% if tools %}
3 + {% set weekday_map = {'Monday': '星期一', 'Tuesday': '星期二', 'Wednesday': '星期三', 'Thursday': '星期四', 'Friday': '星期五', 'Saturday': '星期六', 'Sunday': '星期日'} %}
4 + {% set weekday_cn = weekday_map[strftime_now('%A')] %}
5 + {% set datetime_str = strftime_now('%Y-%m-%d %H:%M:%S') %}
6 + {% set datetime_str = datetime_str + ' ' + weekday_cn %}
7 + {% for message in loop_messages %}
8 + {% if 'content' in message %}
9 + {% set content = message['content'] %}
10 + {% else %}
11 + {% set content = '' %}
12 + {% endif %}
13 + {% if loop.index0 == 0 %}
14 + {% set content_tmp = '你是一位函数组合专家。你会得到一个问题和一组可能的函数。根据问题，你需要进行一个或多个函数/工具调用以实现目的。
15 +如果没有一个函数可以使用，请直接使用自然语言回复用户，以助手：开头。
16 +如果给定的问题缺少函数所需的参数，请使用自然语言进行提问，向用户询问必要信息，以助手：开头。
17 +如果调用结果已经足够回答用户问题，请对历史结果进行总结，使用自然语言回复用户，以助手：开头。
18 +你应该只在工具调用部分返回函数调用。如果你决定调用任何函数，你必须将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>。你不应该在回复中包含任何其他文本。以下是你可以调用的函数列表，格式为JSON。
19 +' %}
20 + {% set content_tmp = content_tmp + '
21 +' + tools | tojson + '
22 +' %}
23 + {% if message['role'] == 'system' %}
24 + {% set content_tmp = content_tmp + '
25 +额外要求：
26 +' + content + '
27 +
28 +如果你决定返回函数调用，请将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>，不得包含其他文本。如果额外要求里有格式要求，请忽略，以此处为准。
29 +否则，请参考开头说的三种情况，以助手：开头进行回复。
30 +
31 +如果额外要求里有时间信息，就以额外要求里的时间为准，否则，参考当前时间：' + datetime_str %}
32 + {% set content = '<|startoftext|>' + content_tmp + '<|extra_4|>' %}
33 + {% elif message['role'] == 'user' %}
34 + {% set content_tmp = content_tmp + '
35 +如果你决定返回函数调用，请将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>，不得包含其他文本。
36 +否则，请参考开头说的三种情况，以助手：开头进行回复。
37 +
38 +当前时间：' + datetime_str %}
39 + {% set content_tmp = '<|startoftext|>' + content_tmp + '<|extra_4|>'%}
40 + {% set content = content_tmp + '用户：' + content + '<|extra_0|>' %}
41 + {% endif %}
42 + {% else %}
43 + {% if message['role'] == 'user' %}
44 + {% set content = '用户：' + content + '<|extra_0|>' %}
45 + {% elif message['role'] == 'assistant' %}
46 + {% if 'tool_calls' in message %}
47 + {% set tool_calls = message['tool_calls'] %}
48 + {% set ns = namespace(tool_calls="[") %}
49 + {% for tool_call in tool_calls %}
50 + {% set function = tool_call['function'] %}
51 + {% set name = function['name'] %}
52 + {% set ns.tool_calls = ns.tool_calls + '{"name": "' + name + '", '%}
53 + {% set arguments = function['arguments'] %}
54 + {% if arguments is not string %}
55 + {% set arguments = arguments | tojson %}
56 + {% endif %}
57 + {% set ns.tool_calls = ns.tool_calls + '"arguments": ' + arguments + '}' %}
58 + {% if not loop.last %}
59 + {% set ns.tool_calls = ns.tool_calls + ', '%}
60 + {% endif %}
61 + {% endfor %}
62 + {% set ns.tool_calls = ns.tool_calls + ']' %}
63 + {% set content = content + '<tool_calls>' + ns.tool_calls + '</tool_calls>' %}
64 + {% else %}
65 + {% set content = '助手：' + content %}
66 + {% endif %}
67 + {% set content = content + '<|eos|>' %}
68 + {% elif message['role'] == 'tool' %}
69 + {% if content is not string %}
70 + {set content = content | tojson }
71 + {% endif %}
72 + {% set content = '<tool_response>' + content + '</tool_response>' %}
73 + {% set content = content + '<|extra_0|>' %}
74 + {% endif %}
75 + {% endif %}
76 + {{- content -}}
77 + {% endfor %}
78 +{% else %}
79 + {% set context = {'has_head': true} %}
80 + {% for message in loop_messages %}
81 + {% if 'content' in message %}
82 + {% set content = message['content'] %}
83 + {% else %}
84 + {% set content = '' %}
85 + {% endif %}
86 + {% if loop.index0 == 0 %}
87 + {% if content == '' %}
88 + {% set _ = context.update({'has_head': false}) %}
89 + {% elif message['role'] == 'system' %}
90 + {% set content = '<|startoftext|>' + content + '<|extra_4|>' %}
91 + {% endif %}
92 + {% endif %}
93 + {% if message['role'] == 'user' %}
94 + {% if loop.index0 == 1 and not context.has_head %}
95 + {% set content = '<|startoftext|>' + content %}
96 + {% endif %}
97 + {% if loop.index0 == 1 and context.has_head %}
98 + {% set content = content + '<|extra_0|>' %}
99 + {% else %}
100 + {% set content = '<|startoftext|>' + content + '<|extra_0|>' %}
101 + {% endif %}
102 + {% elif message['role'] == 'assistant' %}
103 + {% set content = content + '<|eos|>' %}
104 + {% elif message['role'] == 'tool' %}
105 + {% set content = content + '<|extra_0|>' %}
106 + {% endif %}
107 + {{- content -}}
108 + {% endfor %}
109 +{% endif %}
110 +{%- if enable_thinking is defined and enable_thinking is false %}
111 + {{- '<think>\n\n</think>\n' }}
112 +{%- endif %}
113 +
tests/entrypoints/openai/tool_parsers/test_hunyuan_a13b_tool_parser.py new
+153
@@ -0,0 +1,153 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# ruff: noqa: E501
4 +
5 +import json
6 +from unittest.mock import MagicMock
7 +
8 +import pytest
9 +
10 +from tests.entrypoints.openai.tool_parsers.utils import (
11 + run_tool_extraction, run_tool_extraction_streaming)
12 +from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
13 +from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
14 +
15 +
16 +def make_tool_call(name, arguments):
17 + return ToolCall(type="function",
18 + function=FunctionCall(name=name,
19 + arguments=json.dumps(arguments)))
20 +
21 +
22 +# TODO: add reason prefix and suffix.
23 +
24 +
25 +@pytest.mark.parametrize(
26 + "model_output,expected_tool_calls,expected_content",
27 + [
28 + # No tool call
29 + ("How can I help you today?", [], "How can I help you today?"),
30 + # Single tool call, no content
31 + (
32 + "<tool_calls>[{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Francisco\", \"metric\": \"celsius\"}}]</tool_calls>", #noqa: E501
33 + [
34 + make_tool_call("get_weather", {
35 + "city": "San Francisco",
36 + "metric": "celsius"
37 + })
38 + ],
39 + None),
40 + # Multiple tool calls
41 + (
42 + "<tool_calls>[{\"name\": \"get_weather\", \"arguments\": {\"city\": \"San Francisco\", \"metric\": \"celsius\"}}, {\"name\": \"register_user\", \"arguments\": {\"name\": \"John Doe\", \"age\": 37, \"address\": {\"city\": \"San Francisco\", \"state\": \"CA\"}, \"role\": null, \"passed_test\": true, \"aliases\": [\"John\", \"Johnny\"]}}]</tool_calls>", #noqa: E501
43 + [
44 + make_tool_call("get_weather", {
45 + "city": "San Francisco",
46 + "metric": "celsius"
47 + }),
48 + make_tool_call(
49 + "register_user", {
50 + "name": "John Doe",
51 + "age": 37,
52 + "address": {
53 + "city": "San Francisco",
54 + "state": "CA"
55 + },
56 + "role": None,
57 + "passed_test": True,
58 + "aliases": ["John", "Johnny"]
59 + })
60 + ],
61 + None),
62 + # Content before tool call
63 + (
64 + "I will call the tool now. <tool_calls>[{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Boston\"}}]</tool_calls>", #noqa: E501
65 + [make_tool_call("get_weather", {"city": "Boston"})],
66 + "I will call the tool now. "),
67 + # Content after tool call (should be stripped)
68 + (
69 + "<tool_calls>[{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Seattle\"}}]</tool_calls>\nThank you!", #noqa: E501
70 + [make_tool_call("get_weather", {"city": "Seattle"})],
71 + None),
72 + (
73 + "<tool_calls>[{\"name\": \"complex_tool\", \"arguments\": {\"level1\": {\"level2\": {\"level3\": {\"value\": 123}}}}}]</tool_calls>",
74 + [
75 + make_tool_call(
76 + "complex_tool",
77 + {"level1": {
78 + "level2": {
79 + "level3": {
80 + "value": 123
81 + }
82 + }
83 + }})
84 + ],
85 + None,
86 + ),
87 + ])
88 +def test_hunyuan_a13b_tool_parser_extract(model_output, expected_tool_calls,
89 + expected_content):
90 + mock_tokenizer = MagicMock()
91 + tool_parser: ToolParser = ToolParserManager.get_tool_parser(
92 + "hunyuan_a13b")(mock_tokenizer)
93 + content, tool_calls = run_tool_extraction(tool_parser,
94 + model_output,
95 + streaming=False)
96 +
97 + # align the random id.
98 + for idx in range(len(tool_calls)):
99 + tool_calls[idx].id = expected_tool_calls[idx].id
100 + assert tool_calls == expected_tool_calls
101 + assert content == expected_content
102 +
103 +
104 +# Streaming test: simulate incremental output
105 +@pytest.mark.parametrize("model_deltas,expected_tool_calls", [
106 + ([
107 + "<tool_calls>[{\"name\": \"get_weather\", ",
108 + "\"arguments\": {\"city\": \"San Francisco\", ",
109 + "\"metric\": \"celsius\"}}]", "</tool_calls>"
110 + ], [
111 + make_tool_call("get_weather", {
112 + "city": "San Francisco",
113 + "metric": "celsius"
114 + })
115 + ]),
116 + ([
117 + "<tool_calls>[{\"name\":", " \"get_weather\",", " \"arguments\":",
118 + " {\"city\": \"Boston\"}", "}]", "</tool_calls>"
119 + ], [make_tool_call("get_weather", {"city": "Boston"})]),
120 + ([
121 + "", "<tool_calls>[{\"name\":", " \"get_weather\",", " \"arguments\":",
122 + " {\"city\": \"Boston\"}", "}]", "</tool_calls>", "\n</answer>"
123 + ], [make_tool_call("get_weather", {"city": "Boston"})]),
124 + pytest.param([
125 + "<tool_calls>[{\"name\": \"complex_tool\",", " \"arguments\": ",
126 + " {\"level1\": {\"level2\": ", "{\"level3\": {\"value\": 123}}}}}",
127 + "]</tool_calls>"
128 + ], [
129 + make_tool_call("complex_tool",
130 + {"level1": {
131 + "level2": {
132 + "level3": {
133 + "value": 123
134 + }
135 + }
136 + }})
137 + ],
138 + marks=pytest.mark.xfail(
139 + reason="stream parsing not support nested json yet.")),
140 +])
141 +def test_hunyuan_a13b_tool_parser_streaming(model_deltas, expected_tool_calls):
142 + mock_tokenizer = MagicMock()
143 +
144 + tool_parser: ToolParser = ToolParserManager.get_tool_parser(
145 + "hunyuan_a13b")(mock_tokenizer)
146 + reconstructor = run_tool_extraction_streaming(
147 + tool_parser, model_deltas, assert_one_tool_per_delta=False)
148 +
149 + # align the random id.
150 + for idx in range(len(reconstructor.tool_calls)):
151 + reconstructor.tool_calls[idx].id = expected_tool_calls[idx].id
152 +
153 + assert reconstructor.tool_calls == expected_tool_calls
tests/reasoning/test_hunyuan_reasoning_parser.py
+11
@@ -30,6 +30,12 @@ COMPLETE_REASONING = {
30 "reasoning_content": "This is a reasoning section",
31 "content": None,
32 }
33 +
34 +COMPLETE_REASONING_WITH_SYMBOL = {
35 + "output": f"{START_REASONING}This is a reasoning section!{START_RESPONSE}",
36 + "reasoning_content": "This is a reasoning section!",
37 + "content": None,
38 +}
39 NO_REASONING = {
40 "output": "This is content",
41 "reasoning_content": None,
@@ -70,6 +76,11 @@ TEST_CASES = [
76 COMPLETE_REASONING,
77 id="complete_reasoning",
78 ),
79 + pytest.param(
80 + False,
81 + COMPLETE_REASONING_WITH_SYMBOL,
82 + id="complete_reasoning_with_symbol",
83 + ),
84 pytest.param(
85 False,
86 NO_REASONING,
vllm/entrypoints/openai/serving_chat.py
+16 -3
@@ -613,8 +613,13 @@ class OpenAIServingChat(OpenAIServing):
613 previous_text = previous_texts[i]
614 previous_token_ids = all_previous_token_ids[i]
615 current_text = previous_text + delta_text
616 - current_token_ids = previous_token_ids + list(
617 - output.token_ids)
616 +
617 + # avoid the None + list error.
618 + if previous_token_ids:
619 + current_token_ids = previous_token_ids + list(
620 + output.token_ids)
621 + else:
622 + current_token_ids = list(output.token_ids)
623
624 # handle streaming deltas for tools with named tool_choice
625 if tool_choice_function_name:
@@ -1077,9 +1082,17 @@ class OpenAIServingChat(OpenAIServing):
1082 else:
1083 # FOR NOW make it a chat message; we will have to detect
1084 # the type to make it later.
1085 + ret_content = content
1086 +
1087 + # try to use content return from tool parser first,
1088 + # tool parser may do some modify for the content.
1089 + if (tool_call_info.content
1090 + and len(tool_call_info.content) > 0):
1091 + ret_content = tool_call_info.content
1092 +
1093 message = ChatMessage(role=role,
1094 reasoning_content=reasoning_content,
1082 - content=content)
1095 + content=ret_content)
1096
1097 # undetermined case that is still important to handle
1098 else:
vllm/entrypoints/openai/tool_parsers/__init__.py
+2 -1
@@ -6,6 +6,7 @@ from .deepseekv3_tool_parser import DeepSeekV3ToolParser
6 from .granite_20b_fc_tool_parser import Granite20bFCToolParser
7 from .granite_tool_parser import GraniteToolParser
8 from .hermes_tool_parser import Hermes2ProToolParser
9 +from .hunyuan_a13b_tool_parser import HunyuanA13BToolParser
10 from .internlm2_tool_parser import Internlm2ToolParser
11 from .jamba_tool_parser import JambaToolParser
12 from .kimi_k2_tool_parser import KimiK2ToolParser
@@ -23,5 +24,5 @@ __all__ = [
24 "Internlm2ToolParser", "Llama3JsonToolParser", "JambaToolParser",
25 "Llama4PythonicToolParser", "PythonicToolParser", "Phi4MiniJsonToolParser",
26 "DeepSeekV3ToolParser", "xLAMToolParser", "MinimaxToolParser",
26 - "KimiK2ToolParser"
27 + "KimiK2ToolParser", "HunyuanA13BToolParser"
28 ]
vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py new
+372
@@ -0,0 +1,372 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# ruff: noqa: E501, SIM102
4 +
5 +import json
6 +from collections.abc import Sequence
7 +from typing import Any, Optional, Union
8 +
9 +import regex as re
10 +
11 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 + DeltaFunctionCall, DeltaMessage,
13 + DeltaToolCall,
14 + ExtractedToolCallInformation,
15 + FunctionCall, ToolCall)
16 +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
17 + ToolParser, ToolParserManager)
18 +from vllm.entrypoints.openai.tool_parsers.utils import consume_space
19 +from vllm.logger import init_logger
20 +from vllm.transformers_utils.tokenizer import AnyTokenizer
21 +from vllm.utils import random_uuid
22 +
23 +logger = init_logger(__name__)
24 +
25 +
26 +@ToolParserManager.register_module("hunyuan_a13b")
27 +class HunyuanA13BToolParser(ToolParser):
28 +
29 + def __init__(self, tokenizer: AnyTokenizer):
30 + super().__init__(tokenizer)
31 +
32 + # Initialize state for streaming mode
33 + self.prev_tool_calls: list[dict] = []
34 + self.current_tool_id = -1
35 + self.current_tool_name_sent = False
36 + self.streamed_args: list[str] = [
37 + ] # Track arguments sent for each tool
38 +
39 + # For backward compatibility with tests
40 + self.current_tools_sent: list[bool] = []
41 +
42 + # For backward compatibility with serving code
43 + self.prev_tool_call_arr = []
44 +
45 + # Regex patterns for preprocessing
46 + self.answer_tool_calls_pattern = re.compile(
47 + r"<tool_calls>([\s\S]*?)</tool_calls>", re.DOTALL)
48 +
49 + self.tool_name_reg = re.compile(r'"name"\s*:\s*"([^"]+)"')
50 +
51 + self.tool_empty_arg_reg = re.compile(
52 + r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*\{\s*\}')
53 +
54 + # TODO: not support nested json object in fc arguments.
55 + self.tool_non_empty_arg_reg = re.compile(
56 + r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*(\{(?:[^{}]|(?:\{[^{}]*\}))*\})'
57 + )
58 +
59 + self.bot_string = "<tool_calls>"
60 +
61 + # Define streaming state type to be initialized later
62 + self.streaming_state: dict[str, Any] = {
63 + "current_tool_index": -1,
64 + "tool_ids": [],
65 + "sent_tools": [],
66 + }
67 +
68 + def preprocess_model_output(
69 + self, model_output: str) -> tuple[Optional[str], Optional[str]]:
70 + # find the location tool call
71 + for match in self.answer_tool_calls_pattern.finditer(model_output):
72 + start, end = match.span()
73 + # check tool_calls whether in side of <think>
74 + think_regions = [(m.start(), m.end()) for m in re.finditer(
75 + r"<think>(.*?)</think>", model_output, flags=re.DOTALL)]
76 + in_think = any(start > t_start and end < t_end
77 + for t_start, t_end in think_regions)
78 + if not in_think:
79 + content = model_output[:start]
80 + tool_calls_content = match.group(1).strip()
81 + try:
82 + json.loads(tool_calls_content)
83 + return content, tool_calls_content
84 + except Exception:
85 + continue
86 + return model_output, None
87 +
88 + def extract_tool_calls(
89 + self, model_output: str,
90 + request: ChatCompletionRequest) -> ExtractedToolCallInformation:
91 + """
92 + Extract tool calls from a complete model output.
93 + """
94 + try:
95 + # Preprocess the model output
96 + content, potential_tool_calls = self.preprocess_model_output(
97 + model_output)
98 +
99 + if not potential_tool_calls:
100 + # some text should be filtered out for no function call
101 + # this text is in a13b's chat template.
102 + if content:
103 + content = content.replace("助手：", "", 1)
104 + return ExtractedToolCallInformation(tools_called=False,
105 + tool_calls=[],
106 + content=content)
107 +
108 + # Parse the potential tool calls as JSON
109 + tool_calls_data = json.loads(potential_tool_calls)
110 +
111 + # Ensure it's an array
112 + if not isinstance(tool_calls_data, list):
113 + logger.debug("Tool calls data is not an array")
114 + return ExtractedToolCallInformation(
115 + tools_called=False,
116 + tool_calls=[],
117 + content=content or model_output,
118 + )
119 +
120 + tool_calls: list[ToolCall] = []
121 +
122 + for idx, call in enumerate(tool_calls_data):
123 + if (not isinstance(call, dict) or "name" not in call
124 + or "arguments" not in call):
125 + continue
126 +
127 + tool_call = ToolCall(
128 + id=f"call_{random_uuid()}",
129 + type="function",
130 + function=FunctionCall(
131 + name=call["name"],
132 + arguments=(json.dumps(call["arguments"]) if isinstance(
133 + call["arguments"], dict) else call["arguments"]),
134 + ),
135 + )
136 + tool_calls.append(tool_call)
137 +
138 + if not content or len(content.strip()) == 0:
139 + # clear the whitespace content.
140 + content = None
141 +
142 + return ExtractedToolCallInformation(
143 + tools_called=len(tool_calls) > 0,
144 + tool_calls=tool_calls,
145 + content=content,
146 + )
147 +
148 + except Exception:
149 + return ExtractedToolCallInformation(tools_called=False,
150 + tool_calls=[],
151 + content=model_output)
152 +
153 + def extract_tool_calls_streaming(
154 + self,
155 + previous_text: str,
156 + current_text: str,
157 + delta_text: str,
158 + previous_token_ids: Sequence[int],
159 + current_token_ids: Sequence[int],
160 + delta_token_ids: Sequence[int],
161 + request: ChatCompletionRequest,
162 + ) -> Union[DeltaMessage, None]:
163 + """
164 + Extract tool calls for streaming mode.
165 + """
166 +
167 + start_idx = consume_space(0, current_text)
168 + if current_text[start_idx:].startswith(self.bot_string):
169 + start_idx = consume_space(start_idx + len(self.bot_string),
170 + current_text)
171 + if not current_text or start_idx >= len(
172 + current_text) or current_text[start_idx] != '[':
173 + return DeltaMessage(content=delta_text)
174 +
175 + self._try_parse_json_tools(current_text[start_idx:])
176 +
177 + test_delta = self._handle_test_compatibility(current_text)
178 + if test_delta:
179 + return test_delta
180 +
181 + name_matches = list(self.tool_name_reg.finditer(current_text))
182 + tool_count = len(name_matches)
183 + if tool_count == 0:
184 + return None
185 + self._ensure_state_arrays(tool_count)
186 + current_idx = self.streaming_state["current_tool_index"]
187 +
188 + name_delta = self._handle_tool_name_streaming(current_idx, tool_count,
189 + name_matches)
190 + if name_delta:
191 + return name_delta
192 +
193 + args_delta = self._handle_tool_args_streaming(current_text,
194 + current_idx, tool_count)
195 + if args_delta:
196 + return args_delta
197 +
198 + return None
199 +
200 + def _try_parse_json_tools(self, current_text: str):
201 + try:
202 + parsed_tools = json.loads(current_text)
203 + if isinstance(parsed_tools, list):
204 + self.prev_tool_call_arr = parsed_tools
205 + except json.JSONDecodeError:
206 + pass
207 +
208 + def _handle_test_compatibility(self, current_text: str):
209 + if len(self.current_tools_sent) > 0:
210 + if (len(self.current_tools_sent) == 1
211 + and self.current_tools_sent[0] is False):
212 + name_match = self.tool_name_reg.search(current_text)
213 + if name_match:
214 + function_name = name_match.group(1)
215 + tool_id = f"chatcmpl-tool-{random_uuid()}"
216 + delta = DeltaMessage(tool_calls=[
217 + DeltaToolCall(
218 + index=0,
219 + type="function",
220 + id=tool_id,
221 + function=DeltaFunctionCall(
222 + name=function_name).model_dump(
223 + exclude_none=True),
224 + )
225 + ])
226 + self.current_tools_sent = [True]
227 + self.current_tool_id = 0
228 + self.streaming_state["current_tool_index"] = 0
229 + if len(self.streaming_state["sent_tools"]) == 0:
230 + self.streaming_state["sent_tools"].append({
231 + "sent_name":
232 + True,
233 + "sent_arguments_prefix":
234 + False,
235 + "sent_arguments":
236 + "",
237 + })
238 + else:
239 + self.streaming_state["sent_tools"][0][
240 + "sent_name"] = True
241 + self.current_tool_name_sent = True
242 + return delta
243 + return None
244 +
245 + def _ensure_state_arrays(self, tool_count: int):
246 + while len(self.streaming_state["sent_tools"]) < tool_count:
247 + self.streaming_state["sent_tools"].append({
248 + "sent_name": False,
249 + "sent_arguments_prefix": False,
250 + "sent_arguments": "",
251 + })
252 + while len(self.streaming_state["tool_ids"]) < tool_count:
253 + self.streaming_state["tool_ids"].append(None)
254 +
255 + def _handle_tool_name_streaming(self, current_idx: int, tool_count: int,
256 + name_matches):
257 + if current_idx == -1 or current_idx < tool_count - 1:
258 + next_idx = current_idx + 1
259 + if (next_idx < tool_count
260 + and not self.streaming_state["sent_tools"][next_idx]
261 + ["sent_name"]):
262 + self.streaming_state["current_tool_index"] = next_idx
263 + self.current_tool_id = next_idx
264 + current_idx = next_idx
265 + tool_name = name_matches[current_idx].group(1)
266 + tool_id = f"call_{current_idx}_{random_uuid()}"
267 + self.streaming_state["tool_ids"][current_idx] = tool_id
268 + delta = DeltaMessage(tool_calls=[
269 + DeltaToolCall(
270 + index=current_idx,
271 + type="function",
272 + id=tool_id,
273 + function=DeltaFunctionCall(name=tool_name).model_dump(
274 + exclude_none=True),
275 + )
276 + ])
277 + self.streaming_state["sent_tools"][current_idx][
278 + "sent_name"] = True
279 + self.current_tool_name_sent = True
280 + while len(self.streamed_args) <= current_idx:
281 + self.streamed_args.append("")
282 + return delta
283 + return None
284 +
285 + def _handle_tool_args_streaming(self, current_text: str, current_idx: int,
286 + tool_count: int):
287 +
288 + if current_idx >= 0 and current_idx < tool_count:
289 + empty_args_match = self.tool_empty_arg_reg.search(current_text)
290 + if empty_args_match and empty_args_match.start() > 0:
291 + for i in range(tool_count):
292 + if i == current_idx:
293 + if not self.streaming_state["sent_tools"][current_idx][
294 + "sent_arguments_prefix"]:
295 + self.streaming_state["sent_tools"][current_idx][
296 + "sent_arguments_prefix"] = True
297 + self.streaming_state["sent_tools"][current_idx][
298 + "sent_arguments"] = "{}"
299 + while len(self.streamed_args) <= current_idx:
300 + self.streamed_args.append("")
301 + self.streamed_args[current_idx] += "{}"
302 + delta = DeltaMessage(tool_calls=[
303 + DeltaToolCall(
304 + index=current_idx,
305 + function=DeltaFunctionCall(
306 + arguments="{}").model_dump(
307 + exclude_none=True),
308 + )
309 + ])
310 + if current_idx < tool_count - 1:
311 + self.streaming_state["current_tool_index"] += 1
312 + self.current_tool_id = self.streaming_state[
313 + "current_tool_index"]
314 + return delta
315 +
316 + args_matches = list(
317 + self.tool_non_empty_arg_reg.finditer(current_text))
318 + if current_idx < len(args_matches):
319 + args_text = args_matches[current_idx].group(1)
320 + is_last_tool = current_idx == tool_count - 1
321 + if not is_last_tool:
322 + next_tool_pos = current_text.find(
323 + "},{", args_matches[current_idx].start())
324 + if next_tool_pos != -1:
325 + args_end_pos = (next_tool_pos + 1)
326 + args_text = (
327 + current_text[args_matches[current_idx].start(
328 + ):args_end_pos].split('"arguments":')[1].strip())
329 + sent_args = self.streaming_state["sent_tools"][current_idx][
330 + "sent_arguments"]
331 + if not self.streaming_state["sent_tools"][current_idx][
332 + "sent_arguments_prefix"] and args_text.startswith("{"):
333 + self.streaming_state["sent_tools"][current_idx][
334 + "sent_arguments_prefix"] = True
335 + self.streaming_state["sent_tools"][current_idx][
336 + "sent_arguments"] = "{"
337 + while len(self.streamed_args) <= current_idx:
338 + self.streamed_args.append("")
339 + self.streamed_args[current_idx] += "{"
340 + delta = DeltaMessage(tool_calls=[
341 + DeltaToolCall(
342 + index=current_idx,
343 + function=DeltaFunctionCall(
344 + arguments="{").model_dump(exclude_none=True),
345 + )
346 + ])
347 + return delta
348 +
349 + if args_text.startswith(sent_args):
350 + args_diff = args_text[len(sent_args):]
351 + if args_diff:
352 + self.streaming_state["sent_tools"][current_idx][
353 + "sent_arguments"] = args_text
354 + while len(self.streamed_args) <= current_idx:
355 + self.streamed_args.append("")
356 + self.streamed_args[current_idx] += args_diff
357 + delta = DeltaMessage(tool_calls=[
358 + DeltaToolCall(
359 + index=current_idx,
360 + function=DeltaFunctionCall(
361 + arguments=args_diff).model_dump(
362 + exclude_none=True),
363 + )
364 + ])
365 + return delta
366 +
367 + if args_text.endswith("}") and args_text == sent_args:
368 + if current_idx < tool_count - 1:
369 + self.streaming_state["current_tool_index"] += 1
370 + self.current_tool_id = self.streaming_state[
371 + "current_tool_index"]
372 + return None
vllm/model_executor/layers/fused_moe/configs/E=64,N=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 5
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 5
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=64,N=3072,device_name=NVIDIA_H20,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 256,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 256,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 5
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=64,N=3072,device_name=NVIDIA_H20.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 256,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 5
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=64,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=64,N=384,device_name=NVIDIA_H20.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 8,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=64,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 8,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 5
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=64,N=768,device_name=NVIDIA_H20.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 8,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
vllm/reasoning/hunyuan_a13b_reasoning_parser.py
+7
@@ -83,6 +83,13 @@ class HunyuanA13BReasoningParser(ReasoningParser):
83 def is_reasoning_end(self, input_ids: list[int]) -> bool:
84 return self.current_state == "response"
85
86 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:
87 + # for hunyuan streaming reason parsing, the stream parse
88 + # will call first, and the same token will be called in
89 + # is_reasoning_end and extract_content_ids
90 + # this id is not part of content, so just return [] here.
91 + return []
92 +
93 def extract_reasoning_content(
94 self, model_output: str, request: ChatCompletionRequest
95 ) -> tuple[Optional[str], Optional[str]]: