A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix Qwen3-coder moe tuned config (#24072)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Sep 7, 2025 at 13:19 UTC
62f66be1f74378e5a22e266ad161023c324cf4f8
2 files changed
+45
-41
benchmarks/kernels/benchmark_moe.py
+5
-1
@@ -678,7 +678,11 @@ def main(args: argparse.Namespace):
678
is_fp16 = not (use_fp8_w8a8 or use_int8_w8a16)
679
search_space = get_configs_compute_bound(is_fp16, block_quant_shape)
680
print(f"Start tuning over {len(search_space)} configurations...")
681
-
681
+ if use_deep_gemm:
682
+ raise ValueError(
683
+ "Tuning with --use-deep-gemm is not supported as it only tunes Triton "
684
+ "kernels. Please remove the flag."
685
+ )
686
start = time.time()
687
configs = _distribute(
688
"tune",
vllm/model_executor/layers/fused_moe/configs/E=20,N=2560,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128,128].json
+40
-40
@@ -18,18 +18,18 @@
18
"4": {
19
"BLOCK_SIZE_M": 16,
20
"BLOCK_SIZE_N": 128,
21
- "BLOCK_SIZE_K": 128,
22
- "GROUP_SIZE_M": 1,
21
+ "BLOCK_SIZE_K": 256,
22
+ "GROUP_SIZE_M": 32,
23
"num_warps": 4,
24
"num_stages": 3
25
},
26
"8": {
27
"BLOCK_SIZE_M": 16,
28
"BLOCK_SIZE_N": 128,
29
- "BLOCK_SIZE_K": 256,
29
+ "BLOCK_SIZE_K": 128,
30
"GROUP_SIZE_M": 64,
31
"num_warps": 4,
32
- "num_stages": 4
32
+ "num_stages": 3
33
},
34
"16": {
35
"BLOCK_SIZE_M": 16,
@@ -58,7 +58,7 @@
58
"48": {
59
"BLOCK_SIZE_M": 64,
60
"BLOCK_SIZE_N": 128,
61
- "BLOCK_SIZE_K": 128,
61
+ "BLOCK_SIZE_K": 256,
62
"GROUP_SIZE_M": 64,
63
"num_warps": 4,
64
"num_stages": 4
@@ -74,73 +74,73 @@
74
"96": {
75
"BLOCK_SIZE_M": 64,
76
"BLOCK_SIZE_N": 128,
77
- "BLOCK_SIZE_K": 128,
78
- "GROUP_SIZE_M": 16,
77
+ "BLOCK_SIZE_K": 256,
78
+ "GROUP_SIZE_M": 32,
79
"num_warps": 4,
80
- "num_stages": 3
80
+ "num_stages": 4
81
},
82
"128": {
83
- "BLOCK_SIZE_M": 128,
84
- "BLOCK_SIZE_N": 256,
83
+ "BLOCK_SIZE_M": 64,
84
+ "BLOCK_SIZE_N": 128,
85
"BLOCK_SIZE_K": 128,
86
- "GROUP_SIZE_M": 1,
86
+ "GROUP_SIZE_M": 64,
87
"num_warps": 4,
88
- "num_stages": 2
88
+ "num_stages": 4
89
},
90
"256": {
91
- "BLOCK_SIZE_M": 16,
91
+ "BLOCK_SIZE_M": 64,
92
"BLOCK_SIZE_N": 128,
93
"BLOCK_SIZE_K": 256,
94
- "GROUP_SIZE_M": 64,
94
+ "GROUP_SIZE_M": 1,
95
"num_warps": 4,
96
- "num_stages": 3
96
+ "num_stages": 4
97
},
98
"512": {
99
- "BLOCK_SIZE_M": 256,
100
- "BLOCK_SIZE_N": 256,
99
+ "BLOCK_SIZE_M": 64,
100
+ "BLOCK_SIZE_N": 128,
101
"BLOCK_SIZE_K": 256,
102
"GROUP_SIZE_M": 64,
103
- "num_warps": 8,
103
+ "num_warps": 4,
104
"num_stages": 4
105
},
106
"1024": {
107
- "BLOCK_SIZE_M": 256,
108
- "BLOCK_SIZE_N": 256,
107
+ "BLOCK_SIZE_M": 64,
108
+ "BLOCK_SIZE_N": 128,
109
"BLOCK_SIZE_K": 256,
110
- "GROUP_SIZE_M": 16,
110
+ "GROUP_SIZE_M": 64,
111
"num_warps": 4,
112
"num_stages": 4
113
},
114
"1536": {
115
"BLOCK_SIZE_M": 64,
116
- "BLOCK_SIZE_N": 256,
117
- "BLOCK_SIZE_K": 128,
118
- "GROUP_SIZE_M": 1,
119
- "num_warps": 8,
120
- "num_stages": 4
121
- },
122
- "2048": {
123
- "BLOCK_SIZE_M": 32,
116
"BLOCK_SIZE_N": 128,
117
"BLOCK_SIZE_K": 256,
118
"GROUP_SIZE_M": 16,
127
- "num_warps": 8,
128
- "num_stages": 5
119
+ "num_warps": 4,
120
+ "num_stages": 3
121
},
130
- "3072": {
131
- "BLOCK_SIZE_M": 128,
122
+ "2048": {
123
+ "BLOCK_SIZE_M": 64,
124
"BLOCK_SIZE_N": 128,
125
"BLOCK_SIZE_K": 128,
126
"GROUP_SIZE_M": 64,
127
"num_warps": 4,
136
- "num_stages": 4
128
+ "num_stages": 3
129
+ },
130
+ "3072": {
131
+ "BLOCK_SIZE_M": 64,
132
+ "BLOCK_SIZE_N": 128,
133
+ "BLOCK_SIZE_K": 128,
134
+ "GROUP_SIZE_M": 32,
135
+ "num_warps": 4,
136
+ "num_stages": 3
137
},
138
"4096": {
139
- "BLOCK_SIZE_M": 128,
140
- "BLOCK_SIZE_N": 256,
141
- "BLOCK_SIZE_K": 256,
142
- "GROUP_SIZE_M": 64,
143
- "num_warps": 8,
144
- "num_stages": 5
139
+ "BLOCK_SIZE_M": 64,
140
+ "BLOCK_SIZE_N": 128,
141
+ "BLOCK_SIZE_K": 128,
142
+ "GROUP_SIZE_M": 16,
143
+ "num_warps": 4,
144
+ "num_stages": 3
145
}
146
}