A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel][ROCm][AMD] fused_moe Triton configs v2 for mi300X (#5932)
Divakar Verma committed
Jun 27, 2024 at 15:41 UTC
c3dde367f16111b8968948a1f8e1a26bdac6ffdd
4 files changed
+500
-194
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json
+118
-46
@@ -1,128 +1,200 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 64,
5
- "BLOCK_SIZE_K": 128,
4
+ "BLOCK_SIZE_N": 32,
5
+ "BLOCK_SIZE_K": 256,
6
"GROUP_SIZE_M": 1,
7
- "num_stages": 0
7
+ "num_warps": 2,
8
+ "num_stages": 0,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 1
12
},
13
"2": {
14
"BLOCK_SIZE_M": 16,
11
- "BLOCK_SIZE_N": 64,
15
+ "BLOCK_SIZE_N": 16,
16
"BLOCK_SIZE_K": 128,
17
"GROUP_SIZE_M": 1,
14
- "num_stages": 0
18
+ "num_warps": 2,
19
+ "num_stages": 0,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 2
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
18
- "BLOCK_SIZE_N": 64,
26
+ "BLOCK_SIZE_N": 32,
27
"BLOCK_SIZE_K": 256,
20
- "GROUP_SIZE_M": 64,
21
- "num_stages": 1
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 2,
30
+ "num_stages": 0,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 2
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
25
- "BLOCK_SIZE_N": 64,
37
+ "BLOCK_SIZE_N": 16,
38
"BLOCK_SIZE_K": 256,
27
- "GROUP_SIZE_M": 32,
28
- "num_stages": 1
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 1,
41
+ "num_stages": 0,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 2
45
},
46
"16": {
47
"BLOCK_SIZE_M": 16,
32
- "BLOCK_SIZE_N": 64,
48
+ "BLOCK_SIZE_N": 16,
49
"BLOCK_SIZE_K": 256,
34
- "GROUP_SIZE_M": 8,
35
- "num_stages": 1
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 4,
52
+ "num_stages": 0,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 2
56
},
57
"24": {
58
"BLOCK_SIZE_M": 16,
39
- "BLOCK_SIZE_N": 64,
40
- "BLOCK_SIZE_K": 256,
41
- "GROUP_SIZE_M": 64,
42
- "num_stages": 1
59
+ "BLOCK_SIZE_N": 32,
60
+ "BLOCK_SIZE_K": 64,
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 1,
63
+ "num_stages": 0,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 2
67
},
68
"32": {
69
"BLOCK_SIZE_M": 16,
46
- "BLOCK_SIZE_N": 128,
47
- "BLOCK_SIZE_K": 256,
48
- "GROUP_SIZE_M": 8,
49
- "num_stages": 1
70
+ "BLOCK_SIZE_N": 16,
71
+ "BLOCK_SIZE_K": 128,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 2,
74
+ "num_stages": 0,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 1
78
},
79
"48": {
80
"BLOCK_SIZE_M": 16,
53
- "BLOCK_SIZE_N": 64,
81
+ "BLOCK_SIZE_N": 16,
82
"BLOCK_SIZE_K": 128,
55
- "GROUP_SIZE_M": 8,
56
- "num_stages": 0
83
+ "GROUP_SIZE_M": 4,
84
+ "num_warps": 2,
85
+ "num_stages": 0,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 2
89
},
90
"64": {
59
- "BLOCK_SIZE_M": 64,
91
+ "BLOCK_SIZE_M": 32,
92
"BLOCK_SIZE_N": 64,
93
"BLOCK_SIZE_K": 128,
62
- "GROUP_SIZE_M": 8,
63
- "num_stages": 0
94
+ "GROUP_SIZE_M": 4,
95
+ "num_warps": 8,
96
+ "num_stages": 0,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 2
100
},
101
"96": {
102
"BLOCK_SIZE_M": 32,
67
- "BLOCK_SIZE_N": 128,
103
+ "BLOCK_SIZE_N": 32,
104
"BLOCK_SIZE_K": 128,
69
- "GROUP_SIZE_M": 16,
70
- "num_stages": 0
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 0,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 2
111
},
112
"128": {
113
"BLOCK_SIZE_M": 64,
114
"BLOCK_SIZE_N": 64,
75
- "BLOCK_SIZE_K": 128,
76
- "GROUP_SIZE_M": 8,
77
- "num_stages": 0
115
+ "BLOCK_SIZE_K": 64,
116
+ "GROUP_SIZE_M": 4,
117
+ "num_warps": 8,
118
+ "num_stages": 0,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 2
122
},
123
"256": {
124
"BLOCK_SIZE_M": 128,
125
"BLOCK_SIZE_N": 128,
126
"BLOCK_SIZE_K": 64,
83
- "GROUP_SIZE_M": 8,
84
- "num_stages": 0
127
+ "GROUP_SIZE_M": 4,
128
+ "num_warps": 8,
129
+ "num_stages": 0,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 16,
132
+ "kpack": 1
133
},
134
"512": {
87
- "BLOCK_SIZE_M": 256,
135
+ "BLOCK_SIZE_M": 128,
136
"BLOCK_SIZE_N": 128,
137
"BLOCK_SIZE_K": 64,
90
- "GROUP_SIZE_M": 8,
91
- "num_stages": 0
138
+ "GROUP_SIZE_M": 4,
139
+ "num_warps": 8,
140
+ "num_stages": 0,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 16,
143
+ "kpack": 2
144
},
145
"1024": {
146
"BLOCK_SIZE_M": 128,
147
"BLOCK_SIZE_N": 128,
148
"BLOCK_SIZE_K": 64,
149
"GROUP_SIZE_M": 1,
98
- "num_stages": 0
150
+ "num_warps": 8,
151
+ "num_stages": 0,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 32,
154
+ "kpack": 2
155
},
156
"1536": {
157
"BLOCK_SIZE_M": 128,
158
"BLOCK_SIZE_N": 128,
159
"BLOCK_SIZE_K": 64,
160
"GROUP_SIZE_M": 1,
105
- "num_stages": 0
161
+ "num_warps": 8,
162
+ "num_stages": 0,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 2
166
},
167
"2048": {
168
"BLOCK_SIZE_M": 128,
109
- "BLOCK_SIZE_N": 256,
169
+ "BLOCK_SIZE_N": 128,
170
"BLOCK_SIZE_K": 64,
171
"GROUP_SIZE_M": 1,
112
- "num_stages": 0
172
+ "num_warps": 8,
173
+ "num_stages": 0,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 2
177
},
178
"3072": {
179
"BLOCK_SIZE_M": 128,
116
- "BLOCK_SIZE_N": 256,
180
+ "BLOCK_SIZE_N": 128,
181
"BLOCK_SIZE_K": 64,
182
"GROUP_SIZE_M": 1,
119
- "num_stages": 0
183
+ "num_warps": 8,
184
+ "num_stages": 0,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 1
188
},
189
"4096": {
190
"BLOCK_SIZE_M": 128,
191
"BLOCK_SIZE_N": 128,
192
"BLOCK_SIZE_K": 64,
193
"GROUP_SIZE_M": 1,
126
- "num_stages": 0
194
+ "num_warps": 8,
195
+ "num_stages": 0,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 1
199
}
200
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json
+136
-46
@@ -1,110 +1,200 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 64,
5
- "BLOCK_SIZE_K": 128,
6
- "GROUP_SIZE_M": 64
4
+ "BLOCK_SIZE_N": 32,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 2,
8
+ "num_stages": 0,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 2
12
},
13
"2": {
14
"BLOCK_SIZE_M": 16,
10
- "BLOCK_SIZE_N": 128,
11
- "BLOCK_SIZE_K": 32,
12
- "GROUP_SIZE_M": 32
15
+ "BLOCK_SIZE_N": 64,
16
+ "BLOCK_SIZE_K": 128,
17
+ "GROUP_SIZE_M": 1,
18
+ "num_warps": 4,
19
+ "num_stages": 0,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 1
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
16
- "BLOCK_SIZE_N": 32,
17
- "BLOCK_SIZE_K": 64,
18
- "GROUP_SIZE_M": 8
26
+ "BLOCK_SIZE_N": 64,
27
+ "BLOCK_SIZE_K": 128,
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 4,
30
+ "num_stages": 0,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 2
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
22
- "BLOCK_SIZE_N": 64,
37
+ "BLOCK_SIZE_N": 16,
38
"BLOCK_SIZE_K": 256,
24
- "GROUP_SIZE_M": 1
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 2,
41
+ "num_stages": 0,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 2
45
},
46
"16": {
27
- "BLOCK_SIZE_M": 16,
47
+ "BLOCK_SIZE_M": 64,
48
"BLOCK_SIZE_N": 64,
29
- "BLOCK_SIZE_K": 256,
30
- "GROUP_SIZE_M": 1
49
+ "BLOCK_SIZE_K": 64,
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 8,
52
+ "num_stages": 0,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 1
56
},
57
"24": {
33
- "BLOCK_SIZE_M": 32,
34
- "BLOCK_SIZE_N": 128,
35
- "BLOCK_SIZE_K": 128,
36
- "GROUP_SIZE_M": 1
37
- },
38
- "32": {
39
- "BLOCK_SIZE_M": 64,
58
+ "BLOCK_SIZE_M": 16,
59
"BLOCK_SIZE_N": 64,
60
"BLOCK_SIZE_K": 64,
42
- "GROUP_SIZE_M": 8
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 4,
63
+ "num_stages": 0,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 1
67
+ },
68
+ "32": {
69
+ "BLOCK_SIZE_M": 16,
70
+ "BLOCK_SIZE_N": 16,
71
+ "BLOCK_SIZE_K": 256,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 2,
74
+ "num_stages": 0,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 2
78
},
79
"48": {
45
- "BLOCK_SIZE_M": 128,
46
- "BLOCK_SIZE_N": 64,
80
+ "BLOCK_SIZE_M": 16,
81
+ "BLOCK_SIZE_N": 128,
82
"BLOCK_SIZE_K": 64,
48
- "GROUP_SIZE_M": 8
83
+ "GROUP_SIZE_M": 1,
84
+ "num_warps": 8,
85
+ "num_stages": 0,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 2
89
},
90
"64": {
51
- "BLOCK_SIZE_M": 64,
52
- "BLOCK_SIZE_N": 32,
53
- "BLOCK_SIZE_K": 128,
54
- "GROUP_SIZE_M": 1
91
+ "BLOCK_SIZE_M": 16,
92
+ "BLOCK_SIZE_N": 64,
93
+ "BLOCK_SIZE_K": 64,
94
+ "GROUP_SIZE_M": 1,
95
+ "num_warps": 2,
96
+ "num_stages": 0,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 1
100
},
101
"96": {
102
"BLOCK_SIZE_M": 32,
58
- "BLOCK_SIZE_N": 64,
59
- "BLOCK_SIZE_K": 128,
60
- "GROUP_SIZE_M": 8
103
+ "BLOCK_SIZE_N": 32,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 0,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 2
111
},
112
"128": {
113
"BLOCK_SIZE_M": 64,
114
"BLOCK_SIZE_N": 64,
65
- "BLOCK_SIZE_K": 128,
66
- "GROUP_SIZE_M": 32
115
+ "BLOCK_SIZE_K": 64,
116
+ "GROUP_SIZE_M": 4,
117
+ "num_warps": 8,
118
+ "num_stages": 0,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 1
122
},
123
"256": {
69
- "BLOCK_SIZE_M": 32,
124
+ "BLOCK_SIZE_M": 128,
125
"BLOCK_SIZE_N": 128,
126
"BLOCK_SIZE_K": 64,
72
- "GROUP_SIZE_M": 1
127
+ "GROUP_SIZE_M": 4,
128
+ "num_warps": 8,
129
+ "num_stages": 0,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 16,
132
+ "kpack": 1
133
},
134
"512": {
135
"BLOCK_SIZE_M": 64,
136
"BLOCK_SIZE_N": 64,
137
"BLOCK_SIZE_K": 128,
78
- "GROUP_SIZE_M": 1
138
+ "GROUP_SIZE_M": 1,
139
+ "num_warps": 8,
140
+ "num_stages": 0,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 16,
143
+ "kpack": 2
144
},
145
"1024": {
81
- "BLOCK_SIZE_M": 64,
146
+ "BLOCK_SIZE_M": 128,
147
"BLOCK_SIZE_N": 128,
148
"BLOCK_SIZE_K": 64,
84
- "GROUP_SIZE_M": 1
149
+ "GROUP_SIZE_M": 1,
150
+ "num_warps": 8,
151
+ "num_stages": 0,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 16,
154
+ "kpack": 1
155
},
156
"1536": {
87
- "BLOCK_SIZE_M": 64,
88
- "BLOCK_SIZE_N": 64,
89
- "BLOCK_SIZE_K": 128,
90
- "GROUP_SIZE_M": 1
157
+ "BLOCK_SIZE_M": 128,
158
+ "BLOCK_SIZE_N": 128,
159
+ "BLOCK_SIZE_K": 64,
160
+ "GROUP_SIZE_M": 1,
161
+ "num_warps": 8,
162
+ "num_stages": 0,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 1
166
},
167
"2048": {
168
"BLOCK_SIZE_M": 128,
169
"BLOCK_SIZE_N": 128,
170
"BLOCK_SIZE_K": 64,
96
- "GROUP_SIZE_M": 1
171
+ "GROUP_SIZE_M": 1,
172
+ "num_warps": 8,
173
+ "num_stages": 0,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 2
177
},
178
"3072": {
179
"BLOCK_SIZE_M": 128,
180
"BLOCK_SIZE_N": 128,
181
"BLOCK_SIZE_K": 64,
102
- "GROUP_SIZE_M": 1
182
+ "GROUP_SIZE_M": 1,
183
+ "num_warps": 8,
184
+ "num_stages": 0,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 1
188
},
189
"4096": {
190
"BLOCK_SIZE_M": 128,
191
"BLOCK_SIZE_N": 128,
192
"BLOCK_SIZE_K": 64,
108
- "GROUP_SIZE_M": 1
193
+ "GROUP_SIZE_M": 1,
194
+ "num_warps": 8,
195
+ "num_stages": 0,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 1
199
}
200
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json
+122
-50
@@ -1,128 +1,200 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 64,
5
- "BLOCK_SIZE_K": 64,
6
- "GROUP_SIZE_M": 8,
7
- "num_stages": 0
4
+ "BLOCK_SIZE_N": 16,
5
+ "BLOCK_SIZE_K": 128,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 2,
8
+ "num_stages": 0,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 1
12
},
13
"2": {
14
"BLOCK_SIZE_M": 16,
11
- "BLOCK_SIZE_N": 128,
15
+ "BLOCK_SIZE_N": 16,
16
"BLOCK_SIZE_K": 64,
17
"GROUP_SIZE_M": 1,
14
- "num_stages": 0
18
+ "num_warps": 2,
19
+ "num_stages": 0,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 2
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
18
- "BLOCK_SIZE_N": 64,
19
- "BLOCK_SIZE_K": 32,
20
- "GROUP_SIZE_M": 32,
21
- "num_stages": 1
26
+ "BLOCK_SIZE_N": 32,
27
+ "BLOCK_SIZE_K": 256,
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 2,
30
+ "num_stages": 0,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 2
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
37
"BLOCK_SIZE_N": 32,
38
"BLOCK_SIZE_K": 256,
27
- "GROUP_SIZE_M": 8,
28
- "num_stages": 1
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 2,
41
+ "num_stages": 0,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 2
45
},
46
"16": {
31
- "BLOCK_SIZE_M": 32,
32
- "BLOCK_SIZE_N": 128,
33
- "BLOCK_SIZE_K": 128,
34
- "GROUP_SIZE_M": 16,
35
- "num_stages": 1
47
+ "BLOCK_SIZE_M": 16,
48
+ "BLOCK_SIZE_N": 32,
49
+ "BLOCK_SIZE_K": 256,
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 2,
52
+ "num_stages": 0,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 2
56
},
57
"24": {
58
"BLOCK_SIZE_M": 16,
59
"BLOCK_SIZE_N": 64,
40
- "BLOCK_SIZE_K": 256,
41
- "GROUP_SIZE_M": 8,
42
- "num_stages": 1
60
+ "BLOCK_SIZE_K": 64,
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 4,
63
+ "num_stages": 0,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 1
67
},
68
"32": {
69
"BLOCK_SIZE_M": 16,
46
- "BLOCK_SIZE_N": 256,
47
- "BLOCK_SIZE_K": 64,
48
- "GROUP_SIZE_M": 16,
49
- "num_stages": 0
70
+ "BLOCK_SIZE_N": 16,
71
+ "BLOCK_SIZE_K": 256,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 2,
74
+ "num_stages": 0,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 2
78
},
79
"48": {
80
"BLOCK_SIZE_M": 16,
53
- "BLOCK_SIZE_N": 128,
81
+ "BLOCK_SIZE_N": 32,
82
"BLOCK_SIZE_K": 256,
55
- "GROUP_SIZE_M": 16,
56
- "num_stages": 1
83
+ "GROUP_SIZE_M": 1,
84
+ "num_warps": 2,
85
+ "num_stages": 0,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 2
89
},
90
"64": {
59
- "BLOCK_SIZE_M": 64,
60
- "BLOCK_SIZE_N": 64,
61
- "BLOCK_SIZE_K": 64,
62
- "GROUP_SIZE_M": 32,
63
- "num_stages": 0
91
+ "BLOCK_SIZE_M": 32,
92
+ "BLOCK_SIZE_N": 32,
93
+ "BLOCK_SIZE_K": 256,
94
+ "GROUP_SIZE_M": 4,
95
+ "num_warps": 4,
96
+ "num_stages": 0,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 2
100
},
101
"96": {
102
"BLOCK_SIZE_M": 32,
103
"BLOCK_SIZE_N": 32,
68
- "BLOCK_SIZE_K": 64,
69
- "GROUP_SIZE_M": 16,
70
- "num_stages": 0
104
+ "BLOCK_SIZE_K": 128,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 0,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 1
111
},
112
"128": {
113
"BLOCK_SIZE_M": 64,
74
- "BLOCK_SIZE_N": 256,
75
- "BLOCK_SIZE_K": 64,
76
- "GROUP_SIZE_M": 8,
77
- "num_stages": 0
114
+ "BLOCK_SIZE_N": 64,
115
+ "BLOCK_SIZE_K": 128,
116
+ "GROUP_SIZE_M": 4,
117
+ "num_warps": 8,
118
+ "num_stages": 0,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 1
122
},
123
"256": {
124
"BLOCK_SIZE_M": 128,
125
"BLOCK_SIZE_N": 128,
126
"BLOCK_SIZE_K": 64,
83
- "GROUP_SIZE_M": 8,
84
- "num_stages": 0
127
+ "GROUP_SIZE_M": 4,
128
+ "num_warps": 8,
129
+ "num_stages": 0,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 16,
132
+ "kpack": 1
133
},
134
"512": {
135
"BLOCK_SIZE_M": 64,
88
- "BLOCK_SIZE_N": 64,
89
- "BLOCK_SIZE_K": 128,
136
+ "BLOCK_SIZE_N": 128,
137
+ "BLOCK_SIZE_K": 64,
138
"GROUP_SIZE_M": 1,
91
- "num_stages": 0
139
+ "num_warps": 8,
140
+ "num_stages": 0,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 32,
143
+ "kpack": 2
144
},
145
"1024": {
94
- "BLOCK_SIZE_M": 64,
146
+ "BLOCK_SIZE_M": 128,
147
"BLOCK_SIZE_N": 128,
148
"BLOCK_SIZE_K": 64,
149
"GROUP_SIZE_M": 1,
98
- "num_stages": 0
150
+ "num_warps": 8,
151
+ "num_stages": 0,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 16,
154
+ "kpack": 1
155
},
156
"1536": {
157
"BLOCK_SIZE_M": 128,
158
"BLOCK_SIZE_N": 128,
159
"BLOCK_SIZE_K": 64,
160
"GROUP_SIZE_M": 1,
105
- "num_stages": 0
161
+ "num_warps": 8,
162
+ "num_stages": 0,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 2
166
},
167
"2048": {
168
"BLOCK_SIZE_M": 128,
169
"BLOCK_SIZE_N": 128,
170
"BLOCK_SIZE_K": 64,
171
"GROUP_SIZE_M": 1,
112
- "num_stages": 0
172
+ "num_warps": 8,
173
+ "num_stages": 0,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 1
177
},
178
"3072": {
179
"BLOCK_SIZE_M": 128,
180
"BLOCK_SIZE_N": 128,
181
"BLOCK_SIZE_K": 64,
182
"GROUP_SIZE_M": 1,
119
- "num_stages": 0
183
+ "num_warps": 8,
184
+ "num_stages": 0,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 2
188
},
189
"4096": {
190
"BLOCK_SIZE_M": 128,
191
"BLOCK_SIZE_N": 128,
192
"BLOCK_SIZE_K": 64,
193
"GROUP_SIZE_M": 1,
126
- "num_stages": 0
194
+ "num_warps": 8,
195
+ "num_stages": 0,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 1
199
}
200
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json
+124
-52
@@ -1,128 +1,200 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 64,
4
+ "BLOCK_SIZE_N": 16,
5
"BLOCK_SIZE_K": 256,
6
"GROUP_SIZE_M": 1,
7
- "num_stages": 0
7
+ "num_warps": 2,
8
+ "num_stages": 0,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 2
12
},
13
"2": {
14
"BLOCK_SIZE_M": 16,
11
- "BLOCK_SIZE_N": 128,
12
- "BLOCK_SIZE_K": 256,
15
+ "BLOCK_SIZE_N": 64,
16
+ "BLOCK_SIZE_K": 32,
17
"GROUP_SIZE_M": 1,
14
- "num_stages": 1
18
+ "num_warps": 4,
19
+ "num_stages": 0,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 1
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
18
- "BLOCK_SIZE_N": 64,
19
- "BLOCK_SIZE_K": 256,
20
- "GROUP_SIZE_M": 32,
21
- "num_stages": 1
26
+ "BLOCK_SIZE_N": 32,
27
+ "BLOCK_SIZE_K": 128,
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 4,
30
+ "num_stages": 0,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 1
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
25
- "BLOCK_SIZE_N": 64,
37
+ "BLOCK_SIZE_N": 32,
38
"BLOCK_SIZE_K": 256,
27
- "GROUP_SIZE_M": 8,
28
- "num_stages": 1
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 2,
41
+ "num_stages": 0,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 1
45
},
46
"16": {
47
"BLOCK_SIZE_M": 16,
32
- "BLOCK_SIZE_N": 64,
48
+ "BLOCK_SIZE_N": 16,
49
"BLOCK_SIZE_K": 256,
34
- "GROUP_SIZE_M": 8,
35
- "num_stages": 1
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 4,
52
+ "num_stages": 0,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 2
56
},
57
"24": {
38
- "BLOCK_SIZE_M": 16,
39
- "BLOCK_SIZE_N": 64,
40
- "BLOCK_SIZE_K": 256,
41
- "GROUP_SIZE_M": 8,
42
- "num_stages": 1
58
+ "BLOCK_SIZE_M": 32,
59
+ "BLOCK_SIZE_N": 32,
60
+ "BLOCK_SIZE_K": 128,
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 8,
63
+ "num_stages": 0,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 1
67
},
68
"32": {
69
"BLOCK_SIZE_M": 16,
46
- "BLOCK_SIZE_N": 64,
47
- "BLOCK_SIZE_K": 256,
48
- "GROUP_SIZE_M": 16,
49
- "num_stages": 0
70
+ "BLOCK_SIZE_N": 32,
71
+ "BLOCK_SIZE_K": 128,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 2,
74
+ "num_stages": 0,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 2
78
},
79
"48": {
80
"BLOCK_SIZE_M": 16,
53
- "BLOCK_SIZE_N": 64,
54
- "BLOCK_SIZE_K": 256,
55
- "GROUP_SIZE_M": 16,
56
- "num_stages": 0
81
+ "BLOCK_SIZE_N": 32,
82
+ "BLOCK_SIZE_K": 128,
83
+ "GROUP_SIZE_M": 4,
84
+ "num_warps": 2,
85
+ "num_stages": 0,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 1
89
},
90
"64": {
91
"BLOCK_SIZE_M": 32,
60
- "BLOCK_SIZE_N": 128,
61
- "BLOCK_SIZE_K": 256,
62
- "GROUP_SIZE_M": 8,
63
- "num_stages": 1
92
+ "BLOCK_SIZE_N": 32,
93
+ "BLOCK_SIZE_K": 128,
94
+ "GROUP_SIZE_M": 4,
95
+ "num_warps": 4,
96
+ "num_stages": 0,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 2
100
},
101
"96": {
102
"BLOCK_SIZE_M": 32,
67
- "BLOCK_SIZE_N": 128,
103
+ "BLOCK_SIZE_N": 32,
104
"BLOCK_SIZE_K": 128,
69
- "GROUP_SIZE_M": 8,
70
- "num_stages": 0
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 0,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 2
111
},
112
"128": {
113
"BLOCK_SIZE_M": 64,
114
"BLOCK_SIZE_N": 64,
75
- "BLOCK_SIZE_K": 128,
76
- "GROUP_SIZE_M": 8,
77
- "num_stages": 0
115
+ "BLOCK_SIZE_K": 64,
116
+ "GROUP_SIZE_M": 4,
117
+ "num_warps": 8,
118
+ "num_stages": 0,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 1
122
},
123
"256": {
124
"BLOCK_SIZE_M": 128,
81
- "BLOCK_SIZE_N": 64,
125
+ "BLOCK_SIZE_N": 128,
126
"BLOCK_SIZE_K": 64,
83
- "GROUP_SIZE_M": 8,
84
- "num_stages": 0
127
+ "GROUP_SIZE_M": 4,
128
+ "num_warps": 8,
129
+ "num_stages": 0,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 32,
132
+ "kpack": 2
133
},
134
"512": {
87
- "BLOCK_SIZE_M": 256,
135
+ "BLOCK_SIZE_M": 128,
136
"BLOCK_SIZE_N": 128,
137
"BLOCK_SIZE_K": 64,
90
- "GROUP_SIZE_M": 8,
91
- "num_stages": 0
138
+ "GROUP_SIZE_M": 1,
139
+ "num_warps": 8,
140
+ "num_stages": 0,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 16,
143
+ "kpack": 1
144
},
145
"1024": {
146
"BLOCK_SIZE_M": 128,
147
"BLOCK_SIZE_N": 128,
148
"BLOCK_SIZE_K": 64,
149
"GROUP_SIZE_M": 1,
98
- "num_stages": 0
150
+ "num_warps": 8,
151
+ "num_stages": 0,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 16,
154
+ "kpack": 1
155
},
156
"1536": {
157
"BLOCK_SIZE_M": 128,
158
"BLOCK_SIZE_N": 128,
159
"BLOCK_SIZE_K": 64,
160
"GROUP_SIZE_M": 1,
105
- "num_stages": 0
161
+ "num_warps": 8,
162
+ "num_stages": 0,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 2
166
},
167
"2048": {
168
"BLOCK_SIZE_M": 128,
169
"BLOCK_SIZE_N": 128,
170
"BLOCK_SIZE_K": 64,
171
"GROUP_SIZE_M": 1,
112
- "num_stages": 0
172
+ "num_warps": 8,
173
+ "num_stages": 0,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 1
177
},
178
"3072": {
179
"BLOCK_SIZE_M": 128,
180
"BLOCK_SIZE_N": 128,
181
"BLOCK_SIZE_K": 64,
182
"GROUP_SIZE_M": 1,
119
- "num_stages": 0
183
+ "num_warps": 8,
184
+ "num_stages": 0,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 2
188
},
189
"4096": {
122
- "BLOCK_SIZE_M": 256,
123
- "BLOCK_SIZE_N": 256,
124
- "BLOCK_SIZE_K": 32,
190
+ "BLOCK_SIZE_M": 128,
191
+ "BLOCK_SIZE_N": 128,
192
+ "BLOCK_SIZE_K": 64,
193
"GROUP_SIZE_M": 1,
126
- "num_stages": 0
194
+ "num_warps": 8,
195
+ "num_stages": 0,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 1
199
}
200
}