@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c3dde367f1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel][ROCm][AMD] fused_moe Triton configs v2 for mi300X (#5932)

Divakar Verma committed Jun 27, 2024 at 15:41 UTC c3dde367f16111b8968948a1f8e1a26bdac6ffdd
4 files changed +500 -194
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json
+118 -46
@@ -1,128 +1,200 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 64,
5 - "BLOCK_SIZE_K": 128,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 - "num_stages": 0
7 + "num_warps": 2,
8 + "num_stages": 0,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 1
12 },
13 "2": {
14 "BLOCK_SIZE_M": 16,
11 - "BLOCK_SIZE_N": 64,
15 + "BLOCK_SIZE_N": 16,
16 "BLOCK_SIZE_K": 128,
17 "GROUP_SIZE_M": 1,
14 - "num_stages": 0
18 + "num_warps": 2,
19 + "num_stages": 0,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 2
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
18 - "BLOCK_SIZE_N": 64,
26 + "BLOCK_SIZE_N": 32,
27 "BLOCK_SIZE_K": 256,
20 - "GROUP_SIZE_M": 64,
21 - "num_stages": 1
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 2,
30 + "num_stages": 0,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 2
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
25 - "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_N": 16,
38 "BLOCK_SIZE_K": 256,
27 - "GROUP_SIZE_M": 32,
28 - "num_stages": 1
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 1,
41 + "num_stages": 0,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 2
45 },
46 "16": {
47 "BLOCK_SIZE_M": 16,
32 - "BLOCK_SIZE_N": 64,
48 + "BLOCK_SIZE_N": 16,
49 "BLOCK_SIZE_K": 256,
34 - "GROUP_SIZE_M": 8,
35 - "num_stages": 1
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 4,
52 + "num_stages": 0,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 2
56 },
57 "24": {
58 "BLOCK_SIZE_M": 16,
39 - "BLOCK_SIZE_N": 64,
40 - "BLOCK_SIZE_K": 256,
41 - "GROUP_SIZE_M": 64,
42 - "num_stages": 1
59 + "BLOCK_SIZE_N": 32,
60 + "BLOCK_SIZE_K": 64,
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 1,
63 + "num_stages": 0,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 2
67 },
68 "32": {
69 "BLOCK_SIZE_M": 16,
46 - "BLOCK_SIZE_N": 128,
47 - "BLOCK_SIZE_K": 256,
48 - "GROUP_SIZE_M": 8,
49 - "num_stages": 1
70 + "BLOCK_SIZE_N": 16,
71 + "BLOCK_SIZE_K": 128,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 2,
74 + "num_stages": 0,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 1
78 },
79 "48": {
80 "BLOCK_SIZE_M": 16,
53 - "BLOCK_SIZE_N": 64,
81 + "BLOCK_SIZE_N": 16,
82 "BLOCK_SIZE_K": 128,
55 - "GROUP_SIZE_M": 8,
56 - "num_stages": 0
83 + "GROUP_SIZE_M": 4,
84 + "num_warps": 2,
85 + "num_stages": 0,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 2
89 },
90 "64": {
59 - "BLOCK_SIZE_M": 64,
91 + "BLOCK_SIZE_M": 32,
92 "BLOCK_SIZE_N": 64,
93 "BLOCK_SIZE_K": 128,
62 - "GROUP_SIZE_M": 8,
63 - "num_stages": 0
94 + "GROUP_SIZE_M": 4,
95 + "num_warps": 8,
96 + "num_stages": 0,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 2
100 },
101 "96": {
102 "BLOCK_SIZE_M": 32,
67 - "BLOCK_SIZE_N": 128,
103 + "BLOCK_SIZE_N": 32,
104 "BLOCK_SIZE_K": 128,
69 - "GROUP_SIZE_M": 16,
70 - "num_stages": 0
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 0,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 2
111 },
112 "128": {
113 "BLOCK_SIZE_M": 64,
114 "BLOCK_SIZE_N": 64,
75 - "BLOCK_SIZE_K": 128,
76 - "GROUP_SIZE_M": 8,
77 - "num_stages": 0
115 + "BLOCK_SIZE_K": 64,
116 + "GROUP_SIZE_M": 4,
117 + "num_warps": 8,
118 + "num_stages": 0,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 2
122 },
123 "256": {
124 "BLOCK_SIZE_M": 128,
125 "BLOCK_SIZE_N": 128,
126 "BLOCK_SIZE_K": 64,
83 - "GROUP_SIZE_M": 8,
84 - "num_stages": 0
127 + "GROUP_SIZE_M": 4,
128 + "num_warps": 8,
129 + "num_stages": 0,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 16,
132 + "kpack": 1
133 },
134 "512": {
87 - "BLOCK_SIZE_M": 256,
135 + "BLOCK_SIZE_M": 128,
136 "BLOCK_SIZE_N": 128,
137 "BLOCK_SIZE_K": 64,
90 - "GROUP_SIZE_M": 8,
91 - "num_stages": 0
138 + "GROUP_SIZE_M": 4,
139 + "num_warps": 8,
140 + "num_stages": 0,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 16,
143 + "kpack": 2
144 },
145 "1024": {
146 "BLOCK_SIZE_M": 128,
147 "BLOCK_SIZE_N": 128,
148 "BLOCK_SIZE_K": 64,
149 "GROUP_SIZE_M": 1,
98 - "num_stages": 0
150 + "num_warps": 8,
151 + "num_stages": 0,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 32,
154 + "kpack": 2
155 },
156 "1536": {
157 "BLOCK_SIZE_M": 128,
158 "BLOCK_SIZE_N": 128,
159 "BLOCK_SIZE_K": 64,
160 "GROUP_SIZE_M": 1,
105 - "num_stages": 0
161 + "num_warps": 8,
162 + "num_stages": 0,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 2
166 },
167 "2048": {
168 "BLOCK_SIZE_M": 128,
109 - "BLOCK_SIZE_N": 256,
169 + "BLOCK_SIZE_N": 128,
170 "BLOCK_SIZE_K": 64,
171 "GROUP_SIZE_M": 1,
112 - "num_stages": 0
172 + "num_warps": 8,
173 + "num_stages": 0,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 2
177 },
178 "3072": {
179 "BLOCK_SIZE_M": 128,
116 - "BLOCK_SIZE_N": 256,
180 + "BLOCK_SIZE_N": 128,
181 "BLOCK_SIZE_K": 64,
182 "GROUP_SIZE_M": 1,
119 - "num_stages": 0
183 + "num_warps": 8,
184 + "num_stages": 0,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 1
188 },
189 "4096": {
190 "BLOCK_SIZE_M": 128,
191 "BLOCK_SIZE_N": 128,
192 "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
126 - "num_stages": 0
194 + "num_warps": 8,
195 + "num_stages": 0,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 1
199 }
200 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json
+136 -46
@@ -1,110 +1,200 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 64,
5 - "BLOCK_SIZE_K": 128,
6 - "GROUP_SIZE_M": 64
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 2,
8 + "num_stages": 0,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 2
12 },
13 "2": {
14 "BLOCK_SIZE_M": 16,
10 - "BLOCK_SIZE_N": 128,
11 - "BLOCK_SIZE_K": 32,
12 - "GROUP_SIZE_M": 32
15 + "BLOCK_SIZE_N": 64,
16 + "BLOCK_SIZE_K": 128,
17 + "GROUP_SIZE_M": 1,
18 + "num_warps": 4,
19 + "num_stages": 0,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 1
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
16 - "BLOCK_SIZE_N": 32,
17 - "BLOCK_SIZE_K": 64,
18 - "GROUP_SIZE_M": 8
26 + "BLOCK_SIZE_N": 64,
27 + "BLOCK_SIZE_K": 128,
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 4,
30 + "num_stages": 0,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 2
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
22 - "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_N": 16,
38 "BLOCK_SIZE_K": 256,
24 - "GROUP_SIZE_M": 1
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 2,
41 + "num_stages": 0,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 2
45 },
46 "16": {
27 - "BLOCK_SIZE_M": 16,
47 + "BLOCK_SIZE_M": 64,
48 "BLOCK_SIZE_N": 64,
29 - "BLOCK_SIZE_K": 256,
30 - "GROUP_SIZE_M": 1
49 + "BLOCK_SIZE_K": 64,
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 8,
52 + "num_stages": 0,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 1
56 },
57 "24": {
33 - "BLOCK_SIZE_M": 32,
34 - "BLOCK_SIZE_N": 128,
35 - "BLOCK_SIZE_K": 128,
36 - "GROUP_SIZE_M": 1
37 - },
38 - "32": {
39 - "BLOCK_SIZE_M": 64,
58 + "BLOCK_SIZE_M": 16,
59 "BLOCK_SIZE_N": 64,
60 "BLOCK_SIZE_K": 64,
42 - "GROUP_SIZE_M": 8
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 4,
63 + "num_stages": 0,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 1
67 + },
68 + "32": {
69 + "BLOCK_SIZE_M": 16,
70 + "BLOCK_SIZE_N": 16,
71 + "BLOCK_SIZE_K": 256,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 2,
74 + "num_stages": 0,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 2
78 },
79 "48": {
45 - "BLOCK_SIZE_M": 128,
46 - "BLOCK_SIZE_N": 64,
80 + "BLOCK_SIZE_M": 16,
81 + "BLOCK_SIZE_N": 128,
82 "BLOCK_SIZE_K": 64,
48 - "GROUP_SIZE_M": 8
83 + "GROUP_SIZE_M": 1,
84 + "num_warps": 8,
85 + "num_stages": 0,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 2
89 },
90 "64": {
51 - "BLOCK_SIZE_M": 64,
52 - "BLOCK_SIZE_N": 32,
53 - "BLOCK_SIZE_K": 128,
54 - "GROUP_SIZE_M": 1
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 2,
96 + "num_stages": 0,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 1
100 },
101 "96": {
102 "BLOCK_SIZE_M": 32,
58 - "BLOCK_SIZE_N": 64,
59 - "BLOCK_SIZE_K": 128,
60 - "GROUP_SIZE_M": 8
103 + "BLOCK_SIZE_N": 32,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 0,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 2
111 },
112 "128": {
113 "BLOCK_SIZE_M": 64,
114 "BLOCK_SIZE_N": 64,
65 - "BLOCK_SIZE_K": 128,
66 - "GROUP_SIZE_M": 32
115 + "BLOCK_SIZE_K": 64,
116 + "GROUP_SIZE_M": 4,
117 + "num_warps": 8,
118 + "num_stages": 0,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 1
122 },
123 "256": {
69 - "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_M": 128,
125 "BLOCK_SIZE_N": 128,
126 "BLOCK_SIZE_K": 64,
72 - "GROUP_SIZE_M": 1
127 + "GROUP_SIZE_M": 4,
128 + "num_warps": 8,
129 + "num_stages": 0,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 16,
132 + "kpack": 1
133 },
134 "512": {
135 "BLOCK_SIZE_M": 64,
136 "BLOCK_SIZE_N": 64,
137 "BLOCK_SIZE_K": 128,
78 - "GROUP_SIZE_M": 1
138 + "GROUP_SIZE_M": 1,
139 + "num_warps": 8,
140 + "num_stages": 0,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 16,
143 + "kpack": 2
144 },
145 "1024": {
81 - "BLOCK_SIZE_M": 64,
146 + "BLOCK_SIZE_M": 128,
147 "BLOCK_SIZE_N": 128,
148 "BLOCK_SIZE_K": 64,
84 - "GROUP_SIZE_M": 1
149 + "GROUP_SIZE_M": 1,
150 + "num_warps": 8,
151 + "num_stages": 0,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 16,
154 + "kpack": 1
155 },
156 "1536": {
87 - "BLOCK_SIZE_M": 64,
88 - "BLOCK_SIZE_N": 64,
89 - "BLOCK_SIZE_K": 128,
90 - "GROUP_SIZE_M": 1
157 + "BLOCK_SIZE_M": 128,
158 + "BLOCK_SIZE_N": 128,
159 + "BLOCK_SIZE_K": 64,
160 + "GROUP_SIZE_M": 1,
161 + "num_warps": 8,
162 + "num_stages": 0,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 1
166 },
167 "2048": {
168 "BLOCK_SIZE_M": 128,
169 "BLOCK_SIZE_N": 128,
170 "BLOCK_SIZE_K": 64,
96 - "GROUP_SIZE_M": 1
171 + "GROUP_SIZE_M": 1,
172 + "num_warps": 8,
173 + "num_stages": 0,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 2
177 },
178 "3072": {
179 "BLOCK_SIZE_M": 128,
180 "BLOCK_SIZE_N": 128,
181 "BLOCK_SIZE_K": 64,
102 - "GROUP_SIZE_M": 1
182 + "GROUP_SIZE_M": 1,
183 + "num_warps": 8,
184 + "num_stages": 0,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 1
188 },
189 "4096": {
190 "BLOCK_SIZE_M": 128,
191 "BLOCK_SIZE_N": 128,
192 "BLOCK_SIZE_K": 64,
108 - "GROUP_SIZE_M": 1
193 + "GROUP_SIZE_M": 1,
194 + "num_warps": 8,
195 + "num_stages": 0,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 1
199 }
200 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json
+122 -50
@@ -1,128 +1,200 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 64,
5 - "BLOCK_SIZE_K": 64,
6 - "GROUP_SIZE_M": 8,
7 - "num_stages": 0
4 + "BLOCK_SIZE_N": 16,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 2,
8 + "num_stages": 0,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 1
12 },
13 "2": {
14 "BLOCK_SIZE_M": 16,
11 - "BLOCK_SIZE_N": 128,
15 + "BLOCK_SIZE_N": 16,
16 "BLOCK_SIZE_K": 64,
17 "GROUP_SIZE_M": 1,
14 - "num_stages": 0
18 + "num_warps": 2,
19 + "num_stages": 0,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 2
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
18 - "BLOCK_SIZE_N": 64,
19 - "BLOCK_SIZE_K": 32,
20 - "GROUP_SIZE_M": 32,
21 - "num_stages": 1
26 + "BLOCK_SIZE_N": 32,
27 + "BLOCK_SIZE_K": 256,
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 2,
30 + "num_stages": 0,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 2
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
37 "BLOCK_SIZE_N": 32,
38 "BLOCK_SIZE_K": 256,
27 - "GROUP_SIZE_M": 8,
28 - "num_stages": 1
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 2,
41 + "num_stages": 0,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 2
45 },
46 "16": {
31 - "BLOCK_SIZE_M": 32,
32 - "BLOCK_SIZE_N": 128,
33 - "BLOCK_SIZE_K": 128,
34 - "GROUP_SIZE_M": 16,
35 - "num_stages": 1
47 + "BLOCK_SIZE_M": 16,
48 + "BLOCK_SIZE_N": 32,
49 + "BLOCK_SIZE_K": 256,
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 2,
52 + "num_stages": 0,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 2
56 },
57 "24": {
58 "BLOCK_SIZE_M": 16,
59 "BLOCK_SIZE_N": 64,
40 - "BLOCK_SIZE_K": 256,
41 - "GROUP_SIZE_M": 8,
42 - "num_stages": 1
60 + "BLOCK_SIZE_K": 64,
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 4,
63 + "num_stages": 0,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 1
67 },
68 "32": {
69 "BLOCK_SIZE_M": 16,
46 - "BLOCK_SIZE_N": 256,
47 - "BLOCK_SIZE_K": 64,
48 - "GROUP_SIZE_M": 16,
49 - "num_stages": 0
70 + "BLOCK_SIZE_N": 16,
71 + "BLOCK_SIZE_K": 256,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 2,
74 + "num_stages": 0,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 2
78 },
79 "48": {
80 "BLOCK_SIZE_M": 16,
53 - "BLOCK_SIZE_N": 128,
81 + "BLOCK_SIZE_N": 32,
82 "BLOCK_SIZE_K": 256,
55 - "GROUP_SIZE_M": 16,
56 - "num_stages": 1
83 + "GROUP_SIZE_M": 1,
84 + "num_warps": 2,
85 + "num_stages": 0,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 2
89 },
90 "64": {
59 - "BLOCK_SIZE_M": 64,
60 - "BLOCK_SIZE_N": 64,
61 - "BLOCK_SIZE_K": 64,
62 - "GROUP_SIZE_M": 32,
63 - "num_stages": 0
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 256,
94 + "GROUP_SIZE_M": 4,
95 + "num_warps": 4,
96 + "num_stages": 0,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 2
100 },
101 "96": {
102 "BLOCK_SIZE_M": 32,
103 "BLOCK_SIZE_N": 32,
68 - "BLOCK_SIZE_K": 64,
69 - "GROUP_SIZE_M": 16,
70 - "num_stages": 0
104 + "BLOCK_SIZE_K": 128,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 0,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 1
111 },
112 "128": {
113 "BLOCK_SIZE_M": 64,
74 - "BLOCK_SIZE_N": 256,
75 - "BLOCK_SIZE_K": 64,
76 - "GROUP_SIZE_M": 8,
77 - "num_stages": 0
114 + "BLOCK_SIZE_N": 64,
115 + "BLOCK_SIZE_K": 128,
116 + "GROUP_SIZE_M": 4,
117 + "num_warps": 8,
118 + "num_stages": 0,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 1
122 },
123 "256": {
124 "BLOCK_SIZE_M": 128,
125 "BLOCK_SIZE_N": 128,
126 "BLOCK_SIZE_K": 64,
83 - "GROUP_SIZE_M": 8,
84 - "num_stages": 0
127 + "GROUP_SIZE_M": 4,
128 + "num_warps": 8,
129 + "num_stages": 0,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 16,
132 + "kpack": 1
133 },
134 "512": {
135 "BLOCK_SIZE_M": 64,
88 - "BLOCK_SIZE_N": 64,
89 - "BLOCK_SIZE_K": 128,
136 + "BLOCK_SIZE_N": 128,
137 + "BLOCK_SIZE_K": 64,
138 "GROUP_SIZE_M": 1,
91 - "num_stages": 0
139 + "num_warps": 8,
140 + "num_stages": 0,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 32,
143 + "kpack": 2
144 },
145 "1024": {
94 - "BLOCK_SIZE_M": 64,
146 + "BLOCK_SIZE_M": 128,
147 "BLOCK_SIZE_N": 128,
148 "BLOCK_SIZE_K": 64,
149 "GROUP_SIZE_M": 1,
98 - "num_stages": 0
150 + "num_warps": 8,
151 + "num_stages": 0,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 16,
154 + "kpack": 1
155 },
156 "1536": {
157 "BLOCK_SIZE_M": 128,
158 "BLOCK_SIZE_N": 128,
159 "BLOCK_SIZE_K": 64,
160 "GROUP_SIZE_M": 1,
105 - "num_stages": 0
161 + "num_warps": 8,
162 + "num_stages": 0,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 2
166 },
167 "2048": {
168 "BLOCK_SIZE_M": 128,
169 "BLOCK_SIZE_N": 128,
170 "BLOCK_SIZE_K": 64,
171 "GROUP_SIZE_M": 1,
112 - "num_stages": 0
172 + "num_warps": 8,
173 + "num_stages": 0,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 1
177 },
178 "3072": {
179 "BLOCK_SIZE_M": 128,
180 "BLOCK_SIZE_N": 128,
181 "BLOCK_SIZE_K": 64,
182 "GROUP_SIZE_M": 1,
119 - "num_stages": 0
183 + "num_warps": 8,
184 + "num_stages": 0,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 2
188 },
189 "4096": {
190 "BLOCK_SIZE_M": 128,
191 "BLOCK_SIZE_N": 128,
192 "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
126 - "num_stages": 0
194 + "num_warps": 8,
195 + "num_stages": 0,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 1
199 }
200 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json
+124 -52
@@ -1,128 +1,200 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 64,
4 + "BLOCK_SIZE_N": 16,
5 "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 - "num_stages": 0
7 + "num_warps": 2,
8 + "num_stages": 0,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 2
12 },
13 "2": {
14 "BLOCK_SIZE_M": 16,
11 - "BLOCK_SIZE_N": 128,
12 - "BLOCK_SIZE_K": 256,
15 + "BLOCK_SIZE_N": 64,
16 + "BLOCK_SIZE_K": 32,
17 "GROUP_SIZE_M": 1,
14 - "num_stages": 1
18 + "num_warps": 4,
19 + "num_stages": 0,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 1
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
18 - "BLOCK_SIZE_N": 64,
19 - "BLOCK_SIZE_K": 256,
20 - "GROUP_SIZE_M": 32,
21 - "num_stages": 1
26 + "BLOCK_SIZE_N": 32,
27 + "BLOCK_SIZE_K": 128,
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 4,
30 + "num_stages": 0,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 1
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
25 - "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_N": 32,
38 "BLOCK_SIZE_K": 256,
27 - "GROUP_SIZE_M": 8,
28 - "num_stages": 1
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 2,
41 + "num_stages": 0,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 1
45 },
46 "16": {
47 "BLOCK_SIZE_M": 16,
32 - "BLOCK_SIZE_N": 64,
48 + "BLOCK_SIZE_N": 16,
49 "BLOCK_SIZE_K": 256,
34 - "GROUP_SIZE_M": 8,
35 - "num_stages": 1
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 4,
52 + "num_stages": 0,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 2
56 },
57 "24": {
38 - "BLOCK_SIZE_M": 16,
39 - "BLOCK_SIZE_N": 64,
40 - "BLOCK_SIZE_K": 256,
41 - "GROUP_SIZE_M": 8,
42 - "num_stages": 1
58 + "BLOCK_SIZE_M": 32,
59 + "BLOCK_SIZE_N": 32,
60 + "BLOCK_SIZE_K": 128,
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 8,
63 + "num_stages": 0,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 1
67 },
68 "32": {
69 "BLOCK_SIZE_M": 16,
46 - "BLOCK_SIZE_N": 64,
47 - "BLOCK_SIZE_K": 256,
48 - "GROUP_SIZE_M": 16,
49 - "num_stages": 0
70 + "BLOCK_SIZE_N": 32,
71 + "BLOCK_SIZE_K": 128,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 2,
74 + "num_stages": 0,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 2
78 },
79 "48": {
80 "BLOCK_SIZE_M": 16,
53 - "BLOCK_SIZE_N": 64,
54 - "BLOCK_SIZE_K": 256,
55 - "GROUP_SIZE_M": 16,
56 - "num_stages": 0
81 + "BLOCK_SIZE_N": 32,
82 + "BLOCK_SIZE_K": 128,
83 + "GROUP_SIZE_M": 4,
84 + "num_warps": 2,
85 + "num_stages": 0,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 1
89 },
90 "64": {
91 "BLOCK_SIZE_M": 32,
60 - "BLOCK_SIZE_N": 128,
61 - "BLOCK_SIZE_K": 256,
62 - "GROUP_SIZE_M": 8,
63 - "num_stages": 1
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 4,
95 + "num_warps": 4,
96 + "num_stages": 0,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 2
100 },
101 "96": {
102 "BLOCK_SIZE_M": 32,
67 - "BLOCK_SIZE_N": 128,
103 + "BLOCK_SIZE_N": 32,
104 "BLOCK_SIZE_K": 128,
69 - "GROUP_SIZE_M": 8,
70 - "num_stages": 0
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 0,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 2
111 },
112 "128": {
113 "BLOCK_SIZE_M": 64,
114 "BLOCK_SIZE_N": 64,
75 - "BLOCK_SIZE_K": 128,
76 - "GROUP_SIZE_M": 8,
77 - "num_stages": 0
115 + "BLOCK_SIZE_K": 64,
116 + "GROUP_SIZE_M": 4,
117 + "num_warps": 8,
118 + "num_stages": 0,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 1
122 },
123 "256": {
124 "BLOCK_SIZE_M": 128,
81 - "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_N": 128,
126 "BLOCK_SIZE_K": 64,
83 - "GROUP_SIZE_M": 8,
84 - "num_stages": 0
127 + "GROUP_SIZE_M": 4,
128 + "num_warps": 8,
129 + "num_stages": 0,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 32,
132 + "kpack": 2
133 },
134 "512": {
87 - "BLOCK_SIZE_M": 256,
135 + "BLOCK_SIZE_M": 128,
136 "BLOCK_SIZE_N": 128,
137 "BLOCK_SIZE_K": 64,
90 - "GROUP_SIZE_M": 8,
91 - "num_stages": 0
138 + "GROUP_SIZE_M": 1,
139 + "num_warps": 8,
140 + "num_stages": 0,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 16,
143 + "kpack": 1
144 },
145 "1024": {
146 "BLOCK_SIZE_M": 128,
147 "BLOCK_SIZE_N": 128,
148 "BLOCK_SIZE_K": 64,
149 "GROUP_SIZE_M": 1,
98 - "num_stages": 0
150 + "num_warps": 8,
151 + "num_stages": 0,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 16,
154 + "kpack": 1
155 },
156 "1536": {
157 "BLOCK_SIZE_M": 128,
158 "BLOCK_SIZE_N": 128,
159 "BLOCK_SIZE_K": 64,
160 "GROUP_SIZE_M": 1,
105 - "num_stages": 0
161 + "num_warps": 8,
162 + "num_stages": 0,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 2
166 },
167 "2048": {
168 "BLOCK_SIZE_M": 128,
169 "BLOCK_SIZE_N": 128,
170 "BLOCK_SIZE_K": 64,
171 "GROUP_SIZE_M": 1,
112 - "num_stages": 0
172 + "num_warps": 8,
173 + "num_stages": 0,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 1
177 },
178 "3072": {
179 "BLOCK_SIZE_M": 128,
180 "BLOCK_SIZE_N": 128,
181 "BLOCK_SIZE_K": 64,
182 "GROUP_SIZE_M": 1,
119 - "num_stages": 0
183 + "num_warps": 8,
184 + "num_stages": 0,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 2
188 },
189 "4096": {
122 - "BLOCK_SIZE_M": 256,
123 - "BLOCK_SIZE_N": 256,
124 - "BLOCK_SIZE_K": 32,
190 + "BLOCK_SIZE_M": 128,
191 + "BLOCK_SIZE_N": 128,
192 + "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
126 - "num_stages": 0
194 + "num_warps": 8,
195 + "num_stages": 0,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 1
199 }
200 }