A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[ROCm][MoE] MI300 tuned configs Mixtral-8x(7B,22B) | fp16, fp8 (#12408)
Signed-off-by: Divakar Verma <divakar.verma@amd.com>
Divakar Verma committed
Jan 24, 2025 at 22:17 UTC
bf21481ddef2fa9bb96c13ba1f80072abdae3eb7
16 files changed
+2260
-148
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 64,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 16,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 2,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 64,
23
+ "BLOCK_SIZE_K": 256,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 4,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 32,
32
+ "BLOCK_SIZE_K": 256,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 2,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 16,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 256,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 2,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "BLOCK_SIZE_N": 64,
50
+ "BLOCK_SIZE_K": 256,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 2,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 16,
58
+ "BLOCK_SIZE_N": 32,
59
+ "BLOCK_SIZE_K": 256,
60
+ "GROUP_SIZE_M": 4,
61
+ "num_warps": 2,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 16,
67
+ "BLOCK_SIZE_N": 64,
68
+ "BLOCK_SIZE_K": 128,
69
+ "GROUP_SIZE_M": 4,
70
+ "num_warps": 4,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 32,
76
+ "BLOCK_SIZE_N": 64,
77
+ "BLOCK_SIZE_K": 256,
78
+ "GROUP_SIZE_M": 4,
79
+ "num_warps": 2,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 32,
85
+ "BLOCK_SIZE_N": 64,
86
+ "BLOCK_SIZE_K": 256,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 2,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 64,
94
+ "BLOCK_SIZE_N": 128,
95
+ "BLOCK_SIZE_K": 256,
96
+ "GROUP_SIZE_M": 4,
97
+ "num_warps": 8,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 128,
103
+ "BLOCK_SIZE_N": 128,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 8,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 256,
112
+ "BLOCK_SIZE_N": 128,
113
+ "BLOCK_SIZE_K": 128,
114
+ "GROUP_SIZE_M": 4,
115
+ "num_warps": 8,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 128,
121
+ "BLOCK_SIZE_N": 128,
122
+ "BLOCK_SIZE_K": 64,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 4,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 128,
130
+ "BLOCK_SIZE_N": 256,
131
+ "BLOCK_SIZE_K": 128,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 8,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 256,
140
+ "BLOCK_SIZE_K": 128,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 8,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 256,
149
+ "BLOCK_SIZE_K": 128,
150
+ "GROUP_SIZE_M": 1,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 256,
157
+ "BLOCK_SIZE_N": 256,
158
+ "BLOCK_SIZE_K": 64,
159
+ "GROUP_SIZE_M": 1,
160
+ "num_warps": 8,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json
+32
-32
@@ -1,21 +1,21 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 32,
4
+ "BLOCK_SIZE_N": 16,
5
"BLOCK_SIZE_K": 256,
6
"GROUP_SIZE_M": 1,
7
"num_warps": 2,
8
"num_stages": 2,
9
"waves_per_eu": 0,
10
"matrix_instr_nonkdim": 16,
11
- "kpack": 1
11
+ "kpack": 2
12
},
13
"2": {
14
"BLOCK_SIZE_M": 16,
15
"BLOCK_SIZE_N": 16,
16
- "BLOCK_SIZE_K": 128,
16
+ "BLOCK_SIZE_K": 256,
17
"GROUP_SIZE_M": 1,
18
- "num_warps": 2,
18
+ "num_warps": 4,
19
"num_stages": 2,
20
"waves_per_eu": 0,
21
"matrix_instr_nonkdim": 16,
@@ -23,10 +23,10 @@
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
26
- "BLOCK_SIZE_N": 32,
27
- "BLOCK_SIZE_K": 256,
26
+ "BLOCK_SIZE_N": 16,
27
+ "BLOCK_SIZE_K": 128,
28
"GROUP_SIZE_M": 1,
29
- "num_warps": 2,
29
+ "num_warps": 1,
30
"num_stages": 2,
31
"waves_per_eu": 0,
32
"matrix_instr_nonkdim": 16,
@@ -34,10 +34,10 @@
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
37
- "BLOCK_SIZE_N": 16,
38
- "BLOCK_SIZE_K": 256,
37
+ "BLOCK_SIZE_N": 64,
38
+ "BLOCK_SIZE_K": 64,
39
"GROUP_SIZE_M": 1,
40
- "num_warps": 1,
40
+ "num_warps": 2,
41
"num_stages": 2,
42
"waves_per_eu": 0,
43
"matrix_instr_nonkdim": 16,
@@ -48,7 +48,7 @@
48
"BLOCK_SIZE_N": 16,
49
"BLOCK_SIZE_K": 256,
50
"GROUP_SIZE_M": 1,
51
- "num_warps": 4,
51
+ "num_warps": 2,
52
"num_stages": 2,
53
"waves_per_eu": 0,
54
"matrix_instr_nonkdim": 16,
@@ -56,10 +56,10 @@
56
},
57
"24": {
58
"BLOCK_SIZE_M": 16,
59
- "BLOCK_SIZE_N": 32,
60
- "BLOCK_SIZE_K": 64,
59
+ "BLOCK_SIZE_N": 16,
60
+ "BLOCK_SIZE_K": 256,
61
"GROUP_SIZE_M": 1,
62
- "num_warps": 1,
62
+ "num_warps": 2,
63
"num_stages": 2,
64
"waves_per_eu": 0,
65
"matrix_instr_nonkdim": 16,
@@ -67,32 +67,32 @@
67
},
68
"32": {
69
"BLOCK_SIZE_M": 16,
70
- "BLOCK_SIZE_N": 16,
71
- "BLOCK_SIZE_K": 128,
70
+ "BLOCK_SIZE_N": 32,
71
+ "BLOCK_SIZE_K": 256,
72
"GROUP_SIZE_M": 4,
73
"num_warps": 2,
74
"num_stages": 2,
75
"waves_per_eu": 0,
76
"matrix_instr_nonkdim": 16,
77
- "kpack": 1
77
+ "kpack": 2
78
},
79
"48": {
80
"BLOCK_SIZE_M": 16,
81
- "BLOCK_SIZE_N": 16,
81
+ "BLOCK_SIZE_N": 64,
82
"BLOCK_SIZE_K": 128,
83
"GROUP_SIZE_M": 4,
84
- "num_warps": 2,
84
+ "num_warps": 4,
85
"num_stages": 2,
86
"waves_per_eu": 0,
87
"matrix_instr_nonkdim": 16,
88
- "kpack": 2
88
+ "kpack": 1
89
},
90
"64": {
91
"BLOCK_SIZE_M": 32,
92
"BLOCK_SIZE_N": 64,
93
"BLOCK_SIZE_K": 128,
94
"GROUP_SIZE_M": 4,
95
- "num_warps": 8,
95
+ "num_warps": 4,
96
"num_stages": 2,
97
"waves_per_eu": 0,
98
"matrix_instr_nonkdim": 16,
@@ -100,24 +100,24 @@
100
},
101
"96": {
102
"BLOCK_SIZE_M": 32,
103
- "BLOCK_SIZE_N": 32,
104
- "BLOCK_SIZE_K": 128,
103
+ "BLOCK_SIZE_N": 64,
104
+ "BLOCK_SIZE_K": 256,
105
"GROUP_SIZE_M": 4,
106
- "num_warps": 4,
106
+ "num_warps": 8,
107
"num_stages": 2,
108
"waves_per_eu": 0,
109
"matrix_instr_nonkdim": 16,
110
- "kpack": 2
110
+ "kpack": 1
111
},
112
"128": {
113
"BLOCK_SIZE_M": 64,
114
"BLOCK_SIZE_N": 64,
115
- "BLOCK_SIZE_K": 64,
115
+ "BLOCK_SIZE_K": 128,
116
"GROUP_SIZE_M": 4,
117
- "num_warps": 8,
117
+ "num_warps": 4,
118
"num_stages": 2,
119
"waves_per_eu": 0,
120
- "matrix_instr_nonkdim": 16,
120
+ "matrix_instr_nonkdim": 32,
121
"kpack": 2
122
},
123
"256": {
@@ -129,7 +129,7 @@
129
"num_stages": 2,
130
"waves_per_eu": 0,
131
"matrix_instr_nonkdim": 16,
132
- "kpack": 1
132
+ "kpack": 2
133
},
134
"512": {
135
"BLOCK_SIZE_M": 128,
@@ -150,7 +150,7 @@
150
"num_warps": 8,
151
"num_stages": 2,
152
"waves_per_eu": 0,
153
- "matrix_instr_nonkdim": 32,
153
+ "matrix_instr_nonkdim": 16,
154
"kpack": 2
155
},
156
"1536": {
@@ -184,7 +184,7 @@
184
"num_stages": 2,
185
"waves_per_eu": 0,
186
"matrix_instr_nonkdim": 16,
187
- "kpack": 1
187
+ "kpack": 2
188
},
189
"4096": {
190
"BLOCK_SIZE_M": 128,
@@ -195,6 +195,6 @@
195
"num_stages": 2,
196
"waves_per_eu": 0,
197
"matrix_instr_nonkdim": 16,
198
- "kpack": 1
198
+ "kpack": 2
199
}
200
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 16,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 2,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 32,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 2,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 64,
23
+ "BLOCK_SIZE_K": 256,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 2,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 64,
32
+ "BLOCK_SIZE_K": 256,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 2,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 16,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 256,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 2,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "BLOCK_SIZE_N": 64,
50
+ "BLOCK_SIZE_K": 256,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 2,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 16,
58
+ "BLOCK_SIZE_N": 64,
59
+ "BLOCK_SIZE_K": 256,
60
+ "GROUP_SIZE_M": 4,
61
+ "num_warps": 2,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 16,
67
+ "BLOCK_SIZE_N": 64,
68
+ "BLOCK_SIZE_K": 256,
69
+ "GROUP_SIZE_M": 1,
70
+ "num_warps": 4,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 32,
76
+ "BLOCK_SIZE_N": 64,
77
+ "BLOCK_SIZE_K": 128,
78
+ "GROUP_SIZE_M": 4,
79
+ "num_warps": 2,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 32,
85
+ "BLOCK_SIZE_N": 64,
86
+ "BLOCK_SIZE_K": 128,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 2,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 64,
94
+ "BLOCK_SIZE_N": 64,
95
+ "BLOCK_SIZE_K": 128,
96
+ "GROUP_SIZE_M": 4,
97
+ "num_warps": 4,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 64,
103
+ "BLOCK_SIZE_N": 128,
104
+ "BLOCK_SIZE_K": 128,
105
+ "GROUP_SIZE_M": 1,
106
+ "num_warps": 8,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 128,
112
+ "BLOCK_SIZE_N": 128,
113
+ "BLOCK_SIZE_K": 256,
114
+ "GROUP_SIZE_M": 1,
115
+ "num_warps": 8,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 128,
121
+ "BLOCK_SIZE_N": 256,
122
+ "BLOCK_SIZE_K": 128,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 8,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 128,
130
+ "BLOCK_SIZE_N": 256,
131
+ "BLOCK_SIZE_K": 128,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 8,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 256,
140
+ "BLOCK_SIZE_K": 128,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 8,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 256,
149
+ "BLOCK_SIZE_K": 128,
150
+ "GROUP_SIZE_M": 1,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 256,
157
+ "BLOCK_SIZE_N": 256,
158
+ "BLOCK_SIZE_K": 64,
159
+ "GROUP_SIZE_M": 1,
160
+ "num_warps": 8,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI300X.json
new
+200
@@ -0,0 +1,200 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 16,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 2,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 2
12
+ },
13
+ "2": {
14
+ "BLOCK_SIZE_M": 16,
15
+ "BLOCK_SIZE_N": 64,
16
+ "BLOCK_SIZE_K": 128,
17
+ "GROUP_SIZE_M": 1,
18
+ "num_warps": 2,
19
+ "num_stages": 2,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 2
23
+ },
24
+ "4": {
25
+ "BLOCK_SIZE_M": 16,
26
+ "BLOCK_SIZE_N": 64,
27
+ "BLOCK_SIZE_K": 64,
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 2,
30
+ "num_stages": 2,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 2
34
+ },
35
+ "8": {
36
+ "BLOCK_SIZE_M": 16,
37
+ "BLOCK_SIZE_N": 32,
38
+ "BLOCK_SIZE_K": 64,
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 2,
41
+ "num_stages": 2,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 2
45
+ },
46
+ "16": {
47
+ "BLOCK_SIZE_M": 16,
48
+ "BLOCK_SIZE_N": 16,
49
+ "BLOCK_SIZE_K": 256,
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 1,
52
+ "num_stages": 2,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 2
56
+ },
57
+ "24": {
58
+ "BLOCK_SIZE_M": 16,
59
+ "BLOCK_SIZE_N": 64,
60
+ "BLOCK_SIZE_K": 128,
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 2,
63
+ "num_stages": 2,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 2
67
+ },
68
+ "32": {
69
+ "BLOCK_SIZE_M": 16,
70
+ "BLOCK_SIZE_N": 64,
71
+ "BLOCK_SIZE_K": 128,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 2,
74
+ "num_stages": 2,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 2
78
+ },
79
+ "48": {
80
+ "BLOCK_SIZE_M": 32,
81
+ "BLOCK_SIZE_N": 32,
82
+ "BLOCK_SIZE_K": 256,
83
+ "GROUP_SIZE_M": 1,
84
+ "num_warps": 4,
85
+ "num_stages": 2,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 2
89
+ },
90
+ "64": {
91
+ "BLOCK_SIZE_M": 32,
92
+ "BLOCK_SIZE_N": 32,
93
+ "BLOCK_SIZE_K": 256,
94
+ "GROUP_SIZE_M": 4,
95
+ "num_warps": 4,
96
+ "num_stages": 2,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 2
100
+ },
101
+ "96": {
102
+ "BLOCK_SIZE_M": 32,
103
+ "BLOCK_SIZE_N": 32,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 1
111
+ },
112
+ "128": {
113
+ "BLOCK_SIZE_M": 64,
114
+ "BLOCK_SIZE_N": 64,
115
+ "BLOCK_SIZE_K": 128,
116
+ "GROUP_SIZE_M": 4,
117
+ "num_warps": 8,
118
+ "num_stages": 2,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 2
122
+ },
123
+ "256": {
124
+ "BLOCK_SIZE_M": 128,
125
+ "BLOCK_SIZE_N": 128,
126
+ "BLOCK_SIZE_K": 64,
127
+ "GROUP_SIZE_M": 4,
128
+ "num_warps": 8,
129
+ "num_stages": 2,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 16,
132
+ "kpack": 2
133
+ },
134
+ "512": {
135
+ "BLOCK_SIZE_M": 128,
136
+ "BLOCK_SIZE_N": 128,
137
+ "BLOCK_SIZE_K": 64,
138
+ "GROUP_SIZE_M": 1,
139
+ "num_warps": 8,
140
+ "num_stages": 2,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 16,
143
+ "kpack": 2
144
+ },
145
+ "1024": {
146
+ "BLOCK_SIZE_M": 128,
147
+ "BLOCK_SIZE_N": 128,
148
+ "BLOCK_SIZE_K": 64,
149
+ "GROUP_SIZE_M": 4,
150
+ "num_warps": 8,
151
+ "num_stages": 2,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 16,
154
+ "kpack": 2
155
+ },
156
+ "1536": {
157
+ "BLOCK_SIZE_M": 128,
158
+ "BLOCK_SIZE_N": 128,
159
+ "BLOCK_SIZE_K": 64,
160
+ "GROUP_SIZE_M": 4,
161
+ "num_warps": 8,
162
+ "num_stages": 2,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 2
166
+ },
167
+ "2048": {
168
+ "BLOCK_SIZE_M": 128,
169
+ "BLOCK_SIZE_N": 128,
170
+ "BLOCK_SIZE_K": 64,
171
+ "GROUP_SIZE_M": 4,
172
+ "num_warps": 8,
173
+ "num_stages": 2,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 2
177
+ },
178
+ "3072": {
179
+ "BLOCK_SIZE_M": 128,
180
+ "BLOCK_SIZE_N": 128,
181
+ "BLOCK_SIZE_K": 64,
182
+ "GROUP_SIZE_M": 16,
183
+ "num_warps": 8,
184
+ "num_stages": 2,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 2
188
+ },
189
+ "4096": {
190
+ "BLOCK_SIZE_M": 128,
191
+ "BLOCK_SIZE_N": 128,
192
+ "BLOCK_SIZE_K": 64,
193
+ "GROUP_SIZE_M": 16,
194
+ "num_warps": 8,
195
+ "num_stages": 2,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 2
199
+ }
200
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 32,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 64,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 4,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 64,
23
+ "BLOCK_SIZE_K": 128,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 4,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 128,
32
+ "BLOCK_SIZE_K": 128,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 8,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 16,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 128,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 4,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "BLOCK_SIZE_N": 128,
50
+ "BLOCK_SIZE_K": 128,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 8,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 16,
58
+ "BLOCK_SIZE_N": 128,
59
+ "BLOCK_SIZE_K": 128,
60
+ "GROUP_SIZE_M": 4,
61
+ "num_warps": 8,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 16,
67
+ "BLOCK_SIZE_N": 128,
68
+ "BLOCK_SIZE_K": 128,
69
+ "GROUP_SIZE_M": 1,
70
+ "num_warps": 8,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 32,
76
+ "BLOCK_SIZE_N": 128,
77
+ "BLOCK_SIZE_K": 128,
78
+ "GROUP_SIZE_M": 4,
79
+ "num_warps": 4,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 32,
85
+ "BLOCK_SIZE_N": 128,
86
+ "BLOCK_SIZE_K": 128,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 4,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 64,
94
+ "BLOCK_SIZE_N": 128,
95
+ "BLOCK_SIZE_K": 128,
96
+ "GROUP_SIZE_M": 4,
97
+ "num_warps": 8,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 128,
103
+ "BLOCK_SIZE_N": 128,
104
+ "BLOCK_SIZE_K": 128,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 8,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 64,
112
+ "BLOCK_SIZE_N": 128,
113
+ "BLOCK_SIZE_K": 128,
114
+ "GROUP_SIZE_M": 1,
115
+ "num_warps": 4,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 128,
121
+ "BLOCK_SIZE_N": 256,
122
+ "BLOCK_SIZE_K": 128,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 8,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 64,
130
+ "BLOCK_SIZE_N": 128,
131
+ "BLOCK_SIZE_K": 128,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 4,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 256,
140
+ "BLOCK_SIZE_K": 128,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 8,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 128,
149
+ "BLOCK_SIZE_K": 256,
150
+ "GROUP_SIZE_M": 1,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 128,
157
+ "BLOCK_SIZE_N": 256,
158
+ "BLOCK_SIZE_K": 128,
159
+ "GROUP_SIZE_M": 4,
160
+ "num_warps": 8,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json
+43
-43
@@ -1,10 +1,10 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 32,
5
- "BLOCK_SIZE_K": 256,
4
+ "BLOCK_SIZE_N": 16,
5
+ "BLOCK_SIZE_K": 128,
6
"GROUP_SIZE_M": 1,
7
- "num_warps": 2,
7
+ "num_warps": 4,
8
"num_stages": 2,
9
"waves_per_eu": 0,
10
"matrix_instr_nonkdim": 16,
@@ -19,14 +19,14 @@
19
"num_stages": 2,
20
"waves_per_eu": 0,
21
"matrix_instr_nonkdim": 16,
22
- "kpack": 1
22
+ "kpack": 2
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
26
- "BLOCK_SIZE_N": 64,
26
+ "BLOCK_SIZE_N": 128,
27
"BLOCK_SIZE_K": 128,
28
"GROUP_SIZE_M": 1,
29
- "num_warps": 4,
29
+ "num_warps": 8,
30
"num_stages": 2,
31
"waves_per_eu": 0,
32
"matrix_instr_nonkdim": 16,
@@ -34,76 +34,76 @@
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
37
- "BLOCK_SIZE_N": 16,
38
- "BLOCK_SIZE_K": 256,
37
+ "BLOCK_SIZE_N": 64,
38
+ "BLOCK_SIZE_K": 128,
39
"GROUP_SIZE_M": 1,
40
- "num_warps": 2,
40
+ "num_warps": 4,
41
"num_stages": 2,
42
"waves_per_eu": 0,
43
"matrix_instr_nonkdim": 16,
44
- "kpack": 2
44
+ "kpack": 1
45
},
46
"16": {
47
- "BLOCK_SIZE_M": 64,
47
+ "BLOCK_SIZE_M": 16,
48
"BLOCK_SIZE_N": 64,
49
- "BLOCK_SIZE_K": 64,
49
+ "BLOCK_SIZE_K": 128,
50
"GROUP_SIZE_M": 1,
51
- "num_warps": 8,
51
+ "num_warps": 4,
52
"num_stages": 2,
53
"waves_per_eu": 0,
54
"matrix_instr_nonkdim": 16,
55
- "kpack": 1
55
+ "kpack": 2
56
},
57
"24": {
58
"BLOCK_SIZE_M": 16,
59
"BLOCK_SIZE_N": 64,
60
- "BLOCK_SIZE_K": 64,
60
+ "BLOCK_SIZE_K": 128,
61
"GROUP_SIZE_M": 1,
62
"num_warps": 4,
63
"num_stages": 2,
64
"waves_per_eu": 0,
65
"matrix_instr_nonkdim": 16,
66
- "kpack": 1
66
+ "kpack": 2
67
},
68
"32": {
69
"BLOCK_SIZE_M": 16,
70
- "BLOCK_SIZE_N": 16,
71
- "BLOCK_SIZE_K": 256,
70
+ "BLOCK_SIZE_N": 64,
71
+ "BLOCK_SIZE_K": 128,
72
"GROUP_SIZE_M": 4,
73
- "num_warps": 2,
73
+ "num_warps": 4,
74
"num_stages": 2,
75
"waves_per_eu": 0,
76
"matrix_instr_nonkdim": 16,
77
- "kpack": 2
77
+ "kpack": 1
78
},
79
"48": {
80
"BLOCK_SIZE_M": 16,
81
- "BLOCK_SIZE_N": 128,
82
- "BLOCK_SIZE_K": 64,
83
- "GROUP_SIZE_M": 1,
84
- "num_warps": 8,
81
+ "BLOCK_SIZE_N": 64,
82
+ "BLOCK_SIZE_K": 128,
83
+ "GROUP_SIZE_M": 4,
84
+ "num_warps": 2,
85
"num_stages": 2,
86
"waves_per_eu": 0,
87
"matrix_instr_nonkdim": 16,
88
"kpack": 2
89
},
90
"64": {
91
- "BLOCK_SIZE_M": 16,
91
+ "BLOCK_SIZE_M": 32,
92
"BLOCK_SIZE_N": 64,
93
- "BLOCK_SIZE_K": 64,
93
+ "BLOCK_SIZE_K": 128,
94
"GROUP_SIZE_M": 1,
95
- "num_warps": 2,
95
+ "num_warps": 4,
96
"num_stages": 2,
97
"waves_per_eu": 0,
98
"matrix_instr_nonkdim": 16,
99
- "kpack": 1
99
+ "kpack": 2
100
},
101
"96": {
102
"BLOCK_SIZE_M": 32,
103
- "BLOCK_SIZE_N": 32,
104
- "BLOCK_SIZE_K": 256,
105
- "GROUP_SIZE_M": 4,
106
- "num_warps": 4,
103
+ "BLOCK_SIZE_N": 128,
104
+ "BLOCK_SIZE_K": 128,
105
+ "GROUP_SIZE_M": 1,
106
+ "num_warps": 8,
107
"num_stages": 2,
108
"waves_per_eu": 0,
109
"matrix_instr_nonkdim": 16,
@@ -112,24 +112,24 @@
112
"128": {
113
"BLOCK_SIZE_M": 64,
114
"BLOCK_SIZE_N": 64,
115
- "BLOCK_SIZE_K": 64,
115
+ "BLOCK_SIZE_K": 128,
116
"GROUP_SIZE_M": 4,
117
"num_warps": 8,
118
"num_stages": 2,
119
"waves_per_eu": 0,
120
"matrix_instr_nonkdim": 16,
121
- "kpack": 1
121
+ "kpack": 2
122
},
123
"256": {
124
- "BLOCK_SIZE_M": 128,
125
- "BLOCK_SIZE_N": 128,
126
- "BLOCK_SIZE_K": 64,
127
- "GROUP_SIZE_M": 4,
124
+ "BLOCK_SIZE_M": 32,
125
+ "BLOCK_SIZE_N": 64,
126
+ "BLOCK_SIZE_K": 128,
127
+ "GROUP_SIZE_M": 1,
128
"num_warps": 8,
129
"num_stages": 2,
130
"waves_per_eu": 0,
131
"matrix_instr_nonkdim": 16,
132
- "kpack": 1
132
+ "kpack": 2
133
},
134
"512": {
135
"BLOCK_SIZE_M": 64,
@@ -151,7 +151,7 @@
151
"num_stages": 2,
152
"waves_per_eu": 0,
153
"matrix_instr_nonkdim": 16,
154
- "kpack": 1
154
+ "kpack": 2
155
},
156
"1536": {
157
"BLOCK_SIZE_M": 128,
@@ -162,7 +162,7 @@
162
"num_stages": 2,
163
"waves_per_eu": 0,
164
"matrix_instr_nonkdim": 16,
165
- "kpack": 1
165
+ "kpack": 2
166
},
167
"2048": {
168
"BLOCK_SIZE_M": 128,
@@ -184,7 +184,7 @@
184
"num_stages": 2,
185
"waves_per_eu": 0,
186
"matrix_instr_nonkdim": 16,
187
- "kpack": 1
187
+ "kpack": 2
188
},
189
"4096": {
190
"BLOCK_SIZE_M": 128,
@@ -195,6 +195,6 @@
195
"num_stages": 2,
196
"waves_per_eu": 0,
197
"matrix_instr_nonkdim": 16,
198
- "kpack": 1
198
+ "kpack": 2
199
}
200
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 16,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 64,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 4,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 128,
23
+ "BLOCK_SIZE_K": 128,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 8,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 64,
32
+ "BLOCK_SIZE_K": 256,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 4,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 16,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 256,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 4,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "BLOCK_SIZE_N": 64,
50
+ "BLOCK_SIZE_K": 256,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 2,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 16,
58
+ "BLOCK_SIZE_N": 64,
59
+ "BLOCK_SIZE_K": 256,
60
+ "GROUP_SIZE_M": 1,
61
+ "num_warps": 2,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 16,
67
+ "BLOCK_SIZE_N": 64,
68
+ "BLOCK_SIZE_K": 256,
69
+ "GROUP_SIZE_M": 1,
70
+ "num_warps": 4,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 16,
76
+ "BLOCK_SIZE_N": 64,
77
+ "BLOCK_SIZE_K": 256,
78
+ "GROUP_SIZE_M": 1,
79
+ "num_warps": 4,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 32,
85
+ "BLOCK_SIZE_N": 128,
86
+ "BLOCK_SIZE_K": 256,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 4,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 16,
94
+ "BLOCK_SIZE_N": 64,
95
+ "BLOCK_SIZE_K": 256,
96
+ "GROUP_SIZE_M": 1,
97
+ "num_warps": 4,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 64,
103
+ "BLOCK_SIZE_N": 64,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 1,
106
+ "num_warps": 4,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 64,
112
+ "BLOCK_SIZE_N": 64,
113
+ "BLOCK_SIZE_K": 128,
114
+ "GROUP_SIZE_M": 1,
115
+ "num_warps": 4,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 64,
121
+ "BLOCK_SIZE_N": 128,
122
+ "BLOCK_SIZE_K": 128,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 4,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 128,
130
+ "BLOCK_SIZE_N": 128,
131
+ "BLOCK_SIZE_K": 256,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 8,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 256,
140
+ "BLOCK_SIZE_K": 128,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 8,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 256,
149
+ "BLOCK_SIZE_K": 128,
150
+ "GROUP_SIZE_M": 1,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 256,
157
+ "BLOCK_SIZE_N": 256,
158
+ "BLOCK_SIZE_K": 64,
159
+ "GROUP_SIZE_M": 1,
160
+ "num_warps": 8,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI300X.json
new
+200
@@ -0,0 +1,200 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 64,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 1
12
+ },
13
+ "2": {
14
+ "BLOCK_SIZE_M": 16,
15
+ "BLOCK_SIZE_N": 64,
16
+ "BLOCK_SIZE_K": 128,
17
+ "GROUP_SIZE_M": 1,
18
+ "num_warps": 4,
19
+ "num_stages": 2,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 2
23
+ },
24
+ "4": {
25
+ "BLOCK_SIZE_M": 16,
26
+ "BLOCK_SIZE_N": 32,
27
+ "BLOCK_SIZE_K": 128,
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 2,
30
+ "num_stages": 2,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 2
34
+ },
35
+ "8": {
36
+ "BLOCK_SIZE_M": 16,
37
+ "BLOCK_SIZE_N": 64,
38
+ "BLOCK_SIZE_K": 128,
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 4,
41
+ "num_stages": 2,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 2
45
+ },
46
+ "16": {
47
+ "BLOCK_SIZE_M": 32,
48
+ "BLOCK_SIZE_N": 64,
49
+ "BLOCK_SIZE_K": 128,
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 8,
52
+ "num_stages": 2,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 1
56
+ },
57
+ "24": {
58
+ "BLOCK_SIZE_M": 16,
59
+ "BLOCK_SIZE_N": 64,
60
+ "BLOCK_SIZE_K": 128,
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 2,
63
+ "num_stages": 2,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 1
67
+ },
68
+ "32": {
69
+ "BLOCK_SIZE_M": 16,
70
+ "BLOCK_SIZE_N": 64,
71
+ "BLOCK_SIZE_K": 256,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 4,
74
+ "num_stages": 2,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 2
78
+ },
79
+ "48": {
80
+ "BLOCK_SIZE_M": 16,
81
+ "BLOCK_SIZE_N": 64,
82
+ "BLOCK_SIZE_K": 128,
83
+ "GROUP_SIZE_M": 1,
84
+ "num_warps": 2,
85
+ "num_stages": 2,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 2
89
+ },
90
+ "64": {
91
+ "BLOCK_SIZE_M": 32,
92
+ "BLOCK_SIZE_N": 64,
93
+ "BLOCK_SIZE_K": 128,
94
+ "GROUP_SIZE_M": 4,
95
+ "num_warps": 8,
96
+ "num_stages": 2,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 2
100
+ },
101
+ "96": {
102
+ "BLOCK_SIZE_M": 32,
103
+ "BLOCK_SIZE_N": 32,
104
+ "BLOCK_SIZE_K": 128,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 1
111
+ },
112
+ "128": {
113
+ "BLOCK_SIZE_M": 64,
114
+ "BLOCK_SIZE_N": 64,
115
+ "BLOCK_SIZE_K": 256,
116
+ "GROUP_SIZE_M": 4,
117
+ "num_warps": 8,
118
+ "num_stages": 2,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 2
122
+ },
123
+ "256": {
124
+ "BLOCK_SIZE_M": 64,
125
+ "BLOCK_SIZE_N": 64,
126
+ "BLOCK_SIZE_K": 256,
127
+ "GROUP_SIZE_M": 1,
128
+ "num_warps": 8,
129
+ "num_stages": 2,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 16,
132
+ "kpack": 2
133
+ },
134
+ "512": {
135
+ "BLOCK_SIZE_M": 64,
136
+ "BLOCK_SIZE_N": 128,
137
+ "BLOCK_SIZE_K": 128,
138
+ "GROUP_SIZE_M": 1,
139
+ "num_warps": 8,
140
+ "num_stages": 2,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 32,
143
+ "kpack": 2
144
+ },
145
+ "1024": {
146
+ "BLOCK_SIZE_M": 128,
147
+ "BLOCK_SIZE_N": 128,
148
+ "BLOCK_SIZE_K": 64,
149
+ "GROUP_SIZE_M": 1,
150
+ "num_warps": 8,
151
+ "num_stages": 2,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 16,
154
+ "kpack": 2
155
+ },
156
+ "1536": {
157
+ "BLOCK_SIZE_M": 128,
158
+ "BLOCK_SIZE_N": 128,
159
+ "BLOCK_SIZE_K": 64,
160
+ "GROUP_SIZE_M": 1,
161
+ "num_warps": 8,
162
+ "num_stages": 2,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 2
166
+ },
167
+ "2048": {
168
+ "BLOCK_SIZE_M": 128,
169
+ "BLOCK_SIZE_N": 128,
170
+ "BLOCK_SIZE_K": 64,
171
+ "GROUP_SIZE_M": 1,
172
+ "num_warps": 8,
173
+ "num_stages": 2,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 2
177
+ },
178
+ "3072": {
179
+ "BLOCK_SIZE_M": 128,
180
+ "BLOCK_SIZE_N": 128,
181
+ "BLOCK_SIZE_K": 64,
182
+ "GROUP_SIZE_M": 1,
183
+ "num_warps": 8,
184
+ "num_stages": 2,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 2
188
+ },
189
+ "4096": {
190
+ "BLOCK_SIZE_M": 128,
191
+ "BLOCK_SIZE_N": 128,
192
+ "BLOCK_SIZE_K": 64,
193
+ "GROUP_SIZE_M": 1,
194
+ "num_warps": 8,
195
+ "num_stages": 2,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 2
199
+ }
200
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 64,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 64,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 4,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 64,
23
+ "BLOCK_SIZE_K": 256,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 4,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 64,
32
+ "BLOCK_SIZE_K": 256,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 2,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 16,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 256,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 2,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "BLOCK_SIZE_N": 64,
50
+ "BLOCK_SIZE_K": 256,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 4,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 16,
58
+ "BLOCK_SIZE_N": 64,
59
+ "BLOCK_SIZE_K": 256,
60
+ "GROUP_SIZE_M": 1,
61
+ "num_warps": 4,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 16,
67
+ "BLOCK_SIZE_N": 64,
68
+ "BLOCK_SIZE_K": 256,
69
+ "GROUP_SIZE_M": 1,
70
+ "num_warps": 2,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 16,
76
+ "BLOCK_SIZE_N": 32,
77
+ "BLOCK_SIZE_K": 128,
78
+ "GROUP_SIZE_M": 1,
79
+ "num_warps": 2,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 32,
85
+ "BLOCK_SIZE_N": 128,
86
+ "BLOCK_SIZE_K": 128,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 4,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 64,
94
+ "BLOCK_SIZE_N": 128,
95
+ "BLOCK_SIZE_K": 128,
96
+ "GROUP_SIZE_M": 4,
97
+ "num_warps": 8,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 64,
103
+ "BLOCK_SIZE_N": 64,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 1,
106
+ "num_warps": 4,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 64,
112
+ "BLOCK_SIZE_N": 128,
113
+ "BLOCK_SIZE_K": 128,
114
+ "GROUP_SIZE_M": 1,
115
+ "num_warps": 8,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 64,
121
+ "BLOCK_SIZE_N": 128,
122
+ "BLOCK_SIZE_K": 128,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 4,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 128,
130
+ "BLOCK_SIZE_N": 128,
131
+ "BLOCK_SIZE_K": 256,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 8,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 128,
140
+ "BLOCK_SIZE_K": 256,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 8,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 256,
149
+ "BLOCK_SIZE_K": 128,
150
+ "GROUP_SIZE_M": 4,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 128,
157
+ "BLOCK_SIZE_N": 128,
158
+ "BLOCK_SIZE_K": 128,
159
+ "GROUP_SIZE_M": 1,
160
+ "num_warps": 4,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json
+35
-35
@@ -1,21 +1,21 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 16,
5
- "BLOCK_SIZE_K": 128,
4
+ "BLOCK_SIZE_N": 64,
5
+ "BLOCK_SIZE_K": 256,
6
"GROUP_SIZE_M": 1,
7
- "num_warps": 2,
7
+ "num_warps": 4,
8
"num_stages": 2,
9
"waves_per_eu": 0,
10
"matrix_instr_nonkdim": 16,
11
- "kpack": 1
11
+ "kpack": 2
12
},
13
"2": {
14
"BLOCK_SIZE_M": 16,
15
- "BLOCK_SIZE_N": 16,
16
- "BLOCK_SIZE_K": 64,
15
+ "BLOCK_SIZE_N": 64,
16
+ "BLOCK_SIZE_K": 128,
17
"GROUP_SIZE_M": 1,
18
- "num_warps": 2,
18
+ "num_warps": 4,
19
"num_stages": 2,
20
"waves_per_eu": 0,
21
"matrix_instr_nonkdim": 16,
@@ -23,10 +23,10 @@
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
26
- "BLOCK_SIZE_N": 32,
27
- "BLOCK_SIZE_K": 256,
26
+ "BLOCK_SIZE_N": 64,
27
+ "BLOCK_SIZE_K": 128,
28
"GROUP_SIZE_M": 1,
29
- "num_warps": 2,
29
+ "num_warps": 4,
30
"num_stages": 2,
31
"waves_per_eu": 0,
32
"matrix_instr_nonkdim": 16,
@@ -34,7 +34,7 @@
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
37
- "BLOCK_SIZE_N": 32,
37
+ "BLOCK_SIZE_N": 16,
38
"BLOCK_SIZE_K": 256,
39
"GROUP_SIZE_M": 1,
40
"num_warps": 2,
@@ -52,32 +52,32 @@
52
"num_stages": 2,
53
"waves_per_eu": 0,
54
"matrix_instr_nonkdim": 16,
55
- "kpack": 2
55
+ "kpack": 1
56
},
57
"24": {
58
"BLOCK_SIZE_M": 16,
59
- "BLOCK_SIZE_N": 64,
60
- "BLOCK_SIZE_K": 64,
59
+ "BLOCK_SIZE_N": 32,
60
+ "BLOCK_SIZE_K": 256,
61
"GROUP_SIZE_M": 1,
62
- "num_warps": 4,
62
+ "num_warps": 2,
63
"num_stages": 2,
64
"waves_per_eu": 0,
65
"matrix_instr_nonkdim": 16,
66
- "kpack": 1
66
+ "kpack": 2
67
},
68
"32": {
69
"BLOCK_SIZE_M": 16,
70
- "BLOCK_SIZE_N": 16,
70
+ "BLOCK_SIZE_N": 32,
71
"BLOCK_SIZE_K": 256,
72
"GROUP_SIZE_M": 4,
73
"num_warps": 2,
74
"num_stages": 2,
75
"waves_per_eu": 0,
76
"matrix_instr_nonkdim": 16,
77
- "kpack": 2
77
+ "kpack": 1
78
},
79
"48": {
80
- "BLOCK_SIZE_M": 16,
80
+ "BLOCK_SIZE_M": 32,
81
"BLOCK_SIZE_N": 32,
82
"BLOCK_SIZE_K": 256,
83
"GROUP_SIZE_M": 1,
@@ -85,7 +85,7 @@
85
"num_stages": 2,
86
"waves_per_eu": 0,
87
"matrix_instr_nonkdim": 16,
88
- "kpack": 2
88
+ "kpack": 1
89
},
90
"64": {
91
"BLOCK_SIZE_M": 32,
@@ -101,40 +101,40 @@
101
"96": {
102
"BLOCK_SIZE_M": 32,
103
"BLOCK_SIZE_N": 32,
104
- "BLOCK_SIZE_K": 128,
104
+ "BLOCK_SIZE_K": 256,
105
"GROUP_SIZE_M": 4,
106
- "num_warps": 4,
106
+ "num_warps": 2,
107
"num_stages": 2,
108
"waves_per_eu": 0,
109
"matrix_instr_nonkdim": 16,
110
- "kpack": 1
110
+ "kpack": 2
111
},
112
"128": {
113
"BLOCK_SIZE_M": 64,
114
"BLOCK_SIZE_N": 64,
115
- "BLOCK_SIZE_K": 128,
116
- "GROUP_SIZE_M": 4,
115
+ "BLOCK_SIZE_K": 256,
116
+ "GROUP_SIZE_M": 1,
117
"num_warps": 8,
118
"num_stages": 2,
119
"waves_per_eu": 0,
120
"matrix_instr_nonkdim": 16,
121
- "kpack": 1
121
+ "kpack": 2
122
},
123
"256": {
124
- "BLOCK_SIZE_M": 128,
125
- "BLOCK_SIZE_N": 128,
126
- "BLOCK_SIZE_K": 64,
127
- "GROUP_SIZE_M": 4,
124
+ "BLOCK_SIZE_M": 64,
125
+ "BLOCK_SIZE_N": 64,
126
+ "BLOCK_SIZE_K": 128,
127
+ "GROUP_SIZE_M": 1,
128
"num_warps": 8,
129
"num_stages": 2,
130
"waves_per_eu": 0,
131
"matrix_instr_nonkdim": 16,
132
- "kpack": 1
132
+ "kpack": 2
133
},
134
"512": {
135
"BLOCK_SIZE_M": 64,
136
"BLOCK_SIZE_N": 128,
137
- "BLOCK_SIZE_K": 64,
137
+ "BLOCK_SIZE_K": 128,
138
"GROUP_SIZE_M": 1,
139
"num_warps": 8,
140
"num_stages": 2,
@@ -151,7 +151,7 @@
151
"num_stages": 2,
152
"waves_per_eu": 0,
153
"matrix_instr_nonkdim": 16,
154
- "kpack": 1
154
+ "kpack": 2
155
},
156
"1536": {
157
"BLOCK_SIZE_M": 128,
@@ -173,7 +173,7 @@
173
"num_stages": 2,
174
"waves_per_eu": 0,
175
"matrix_instr_nonkdim": 16,
176
- "kpack": 1
176
+ "kpack": 2
177
},
178
"3072": {
179
"BLOCK_SIZE_M": 128,
@@ -195,6 +195,6 @@
195
"num_stages": 2,
196
"waves_per_eu": 0,
197
"matrix_instr_nonkdim": 16,
198
- "kpack": 1
198
+ "kpack": 2
199
}
200
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 64,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 128,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 8,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 64,
23
+ "BLOCK_SIZE_K": 256,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 2,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 64,
32
+ "BLOCK_SIZE_K": 256,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 4,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 16,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 128,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 4,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "BLOCK_SIZE_N": 64,
50
+ "BLOCK_SIZE_K": 128,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 4,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 16,
58
+ "BLOCK_SIZE_N": 64,
59
+ "BLOCK_SIZE_K": 128,
60
+ "GROUP_SIZE_M": 1,
61
+ "num_warps": 2,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 16,
67
+ "BLOCK_SIZE_N": 128,
68
+ "BLOCK_SIZE_K": 128,
69
+ "GROUP_SIZE_M": 1,
70
+ "num_warps": 8,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 32,
76
+ "BLOCK_SIZE_N": 64,
77
+ "BLOCK_SIZE_K": 256,
78
+ "GROUP_SIZE_M": 1,
79
+ "num_warps": 2,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 32,
85
+ "BLOCK_SIZE_N": 16,
86
+ "BLOCK_SIZE_K": 256,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 4,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 64,
94
+ "BLOCK_SIZE_N": 64,
95
+ "BLOCK_SIZE_K": 128,
96
+ "GROUP_SIZE_M": 4,
97
+ "num_warps": 4,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 64,
103
+ "BLOCK_SIZE_N": 128,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 1,
106
+ "num_warps": 8,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 64,
112
+ "BLOCK_SIZE_N": 128,
113
+ "BLOCK_SIZE_K": 128,
114
+ "GROUP_SIZE_M": 1,
115
+ "num_warps": 8,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 128,
121
+ "BLOCK_SIZE_N": 128,
122
+ "BLOCK_SIZE_K": 256,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 8,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 128,
130
+ "BLOCK_SIZE_N": 128,
131
+ "BLOCK_SIZE_K": 256,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 8,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 128,
140
+ "BLOCK_SIZE_K": 128,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 4,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 256,
149
+ "BLOCK_SIZE_K": 128,
150
+ "GROUP_SIZE_M": 1,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 256,
157
+ "BLOCK_SIZE_N": 256,
158
+ "BLOCK_SIZE_K": 64,
159
+ "GROUP_SIZE_M": 1,
160
+ "num_warps": 8,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X.json
new
+200
@@ -0,0 +1,200 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 16,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 2,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 2
12
+ },
13
+ "2": {
14
+ "BLOCK_SIZE_M": 16,
15
+ "BLOCK_SIZE_N": 16,
16
+ "BLOCK_SIZE_K": 128,
17
+ "GROUP_SIZE_M": 1,
18
+ "num_warps": 2,
19
+ "num_stages": 2,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 2
23
+ },
24
+ "4": {
25
+ "BLOCK_SIZE_M": 16,
26
+ "BLOCK_SIZE_N": 16,
27
+ "BLOCK_SIZE_K": 256,
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 4,
30
+ "num_stages": 2,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 1
34
+ },
35
+ "8": {
36
+ "BLOCK_SIZE_M": 16,
37
+ "BLOCK_SIZE_N": 16,
38
+ "BLOCK_SIZE_K": 256,
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 4,
41
+ "num_stages": 2,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 2
45
+ },
46
+ "16": {
47
+ "BLOCK_SIZE_M": 16,
48
+ "BLOCK_SIZE_N": 16,
49
+ "BLOCK_SIZE_K": 256,
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 4,
52
+ "num_stages": 2,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 2
56
+ },
57
+ "24": {
58
+ "BLOCK_SIZE_M": 16,
59
+ "BLOCK_SIZE_N": 16,
60
+ "BLOCK_SIZE_K": 256,
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 4,
63
+ "num_stages": 2,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 1
67
+ },
68
+ "32": {
69
+ "BLOCK_SIZE_M": 16,
70
+ "BLOCK_SIZE_N": 32,
71
+ "BLOCK_SIZE_K": 256,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 2,
74
+ "num_stages": 2,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 2
78
+ },
79
+ "48": {
80
+ "BLOCK_SIZE_M": 32,
81
+ "BLOCK_SIZE_N": 64,
82
+ "BLOCK_SIZE_K": 64,
83
+ "GROUP_SIZE_M": 1,
84
+ "num_warps": 4,
85
+ "num_stages": 2,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 2
89
+ },
90
+ "64": {
91
+ "BLOCK_SIZE_M": 32,
92
+ "BLOCK_SIZE_N": 32,
93
+ "BLOCK_SIZE_K": 256,
94
+ "GROUP_SIZE_M": 4,
95
+ "num_warps": 4,
96
+ "num_stages": 2,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 1
100
+ },
101
+ "96": {
102
+ "BLOCK_SIZE_M": 32,
103
+ "BLOCK_SIZE_N": 64,
104
+ "BLOCK_SIZE_K": 64,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 2
111
+ },
112
+ "128": {
113
+ "BLOCK_SIZE_M": 64,
114
+ "BLOCK_SIZE_N": 64,
115
+ "BLOCK_SIZE_K": 64,
116
+ "GROUP_SIZE_M": 1,
117
+ "num_warps": 4,
118
+ "num_stages": 2,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 1
122
+ },
123
+ "256": {
124
+ "BLOCK_SIZE_M": 64,
125
+ "BLOCK_SIZE_N": 128,
126
+ "BLOCK_SIZE_K": 128,
127
+ "GROUP_SIZE_M": 1,
128
+ "num_warps": 8,
129
+ "num_stages": 2,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 32,
132
+ "kpack": 2
133
+ },
134
+ "512": {
135
+ "BLOCK_SIZE_M": 128,
136
+ "BLOCK_SIZE_N": 128,
137
+ "BLOCK_SIZE_K": 64,
138
+ "GROUP_SIZE_M": 1,
139
+ "num_warps": 8,
140
+ "num_stages": 2,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 16,
143
+ "kpack": 2
144
+ },
145
+ "1024": {
146
+ "BLOCK_SIZE_M": 128,
147
+ "BLOCK_SIZE_N": 128,
148
+ "BLOCK_SIZE_K": 64,
149
+ "GROUP_SIZE_M": 1,
150
+ "num_warps": 8,
151
+ "num_stages": 2,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 16,
154
+ "kpack": 2
155
+ },
156
+ "1536": {
157
+ "BLOCK_SIZE_M": 128,
158
+ "BLOCK_SIZE_N": 128,
159
+ "BLOCK_SIZE_K": 64,
160
+ "GROUP_SIZE_M": 1,
161
+ "num_warps": 8,
162
+ "num_stages": 2,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 2
166
+ },
167
+ "2048": {
168
+ "BLOCK_SIZE_M": 128,
169
+ "BLOCK_SIZE_N": 128,
170
+ "BLOCK_SIZE_K": 64,
171
+ "GROUP_SIZE_M": 1,
172
+ "num_warps": 8,
173
+ "num_stages": 2,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 2
177
+ },
178
+ "3072": {
179
+ "BLOCK_SIZE_M": 128,
180
+ "BLOCK_SIZE_N": 128,
181
+ "BLOCK_SIZE_K": 64,
182
+ "GROUP_SIZE_M": 1,
183
+ "num_warps": 8,
184
+ "num_stages": 2,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 2
188
+ },
189
+ "4096": {
190
+ "BLOCK_SIZE_M": 128,
191
+ "BLOCK_SIZE_N": 128,
192
+ "BLOCK_SIZE_K": 64,
193
+ "GROUP_SIZE_M": 1,
194
+ "num_warps": 8,
195
+ "num_stages": 2,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 2
199
+ }
200
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 64,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 4,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 64,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 4,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 64,
23
+ "BLOCK_SIZE_K": 128,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 4,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 64,
32
+ "BLOCK_SIZE_K": 256,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 4,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 32,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 256,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 2,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 32,
49
+ "BLOCK_SIZE_N": 64,
50
+ "BLOCK_SIZE_K": 256,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 2,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 32,
58
+ "BLOCK_SIZE_N": 64,
59
+ "BLOCK_SIZE_K": 256,
60
+ "GROUP_SIZE_M": 1,
61
+ "num_warps": 2,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 32,
67
+ "BLOCK_SIZE_N": 128,
68
+ "BLOCK_SIZE_K": 128,
69
+ "GROUP_SIZE_M": 1,
70
+ "num_warps": 4,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 32,
76
+ "BLOCK_SIZE_N": 64,
77
+ "BLOCK_SIZE_K": 256,
78
+ "GROUP_SIZE_M": 4,
79
+ "num_warps": 2,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 64,
85
+ "BLOCK_SIZE_N": 64,
86
+ "BLOCK_SIZE_K": 256,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 4,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 64,
94
+ "BLOCK_SIZE_N": 64,
95
+ "BLOCK_SIZE_K": 256,
96
+ "GROUP_SIZE_M": 4,
97
+ "num_warps": 4,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 128,
103
+ "BLOCK_SIZE_N": 128,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 8,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 64,
112
+ "BLOCK_SIZE_N": 128,
113
+ "BLOCK_SIZE_K": 128,
114
+ "GROUP_SIZE_M": 1,
115
+ "num_warps": 8,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 128,
121
+ "BLOCK_SIZE_N": 128,
122
+ "BLOCK_SIZE_K": 128,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 4,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 128,
130
+ "BLOCK_SIZE_N": 128,
131
+ "BLOCK_SIZE_K": 256,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 8,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 256,
140
+ "BLOCK_SIZE_K": 128,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 8,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 256,
149
+ "BLOCK_SIZE_K": 128,
150
+ "GROUP_SIZE_M": 1,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 128,
157
+ "BLOCK_SIZE_N": 256,
158
+ "BLOCK_SIZE_K": 128,
159
+ "GROUP_SIZE_M": 1,
160
+ "num_warps": 8,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json
+38
-38
@@ -1,7 +1,7 @@
1
{
2
"1": {
3
"BLOCK_SIZE_M": 16,
4
- "BLOCK_SIZE_N": 16,
4
+ "BLOCK_SIZE_N": 32,
5
"BLOCK_SIZE_K": 256,
6
"GROUP_SIZE_M": 1,
7
"num_warps": 2,
@@ -12,54 +12,54 @@
12
},
13
"2": {
14
"BLOCK_SIZE_M": 16,
15
- "BLOCK_SIZE_N": 64,
16
- "BLOCK_SIZE_K": 32,
15
+ "BLOCK_SIZE_N": 16,
16
+ "BLOCK_SIZE_K": 256,
17
"GROUP_SIZE_M": 1,
18
- "num_warps": 4,
18
+ "num_warps": 2,
19
"num_stages": 2,
20
"waves_per_eu": 0,
21
"matrix_instr_nonkdim": 16,
22
- "kpack": 1
22
+ "kpack": 2
23
},
24
"4": {
25
"BLOCK_SIZE_M": 16,
26
- "BLOCK_SIZE_N": 32,
27
- "BLOCK_SIZE_K": 128,
26
+ "BLOCK_SIZE_N": 16,
27
+ "BLOCK_SIZE_K": 256,
28
"GROUP_SIZE_M": 1,
29
- "num_warps": 4,
29
+ "num_warps": 2,
30
"num_stages": 2,
31
"waves_per_eu": 0,
32
"matrix_instr_nonkdim": 16,
33
- "kpack": 1
33
+ "kpack": 2
34
},
35
"8": {
36
"BLOCK_SIZE_M": 16,
37
- "BLOCK_SIZE_N": 32,
37
+ "BLOCK_SIZE_N": 16,
38
"BLOCK_SIZE_K": 256,
39
"GROUP_SIZE_M": 1,
40
- "num_warps": 2,
40
+ "num_warps": 1,
41
"num_stages": 2,
42
"waves_per_eu": 0,
43
"matrix_instr_nonkdim": 16,
44
- "kpack": 1
44
+ "kpack": 2
45
},
46
"16": {
47
"BLOCK_SIZE_M": 16,
48
- "BLOCK_SIZE_N": 16,
49
- "BLOCK_SIZE_K": 256,
48
+ "BLOCK_SIZE_N": 64,
49
+ "BLOCK_SIZE_K": 64,
50
"GROUP_SIZE_M": 1,
51
- "num_warps": 4,
51
+ "num_warps": 2,
52
"num_stages": 2,
53
"waves_per_eu": 0,
54
"matrix_instr_nonkdim": 16,
55
- "kpack": 2
55
+ "kpack": 1
56
},
57
"24": {
58
- "BLOCK_SIZE_M": 32,
59
- "BLOCK_SIZE_N": 32,
60
- "BLOCK_SIZE_K": 128,
58
+ "BLOCK_SIZE_M": 16,
59
+ "BLOCK_SIZE_N": 16,
60
+ "BLOCK_SIZE_K": 256,
61
"GROUP_SIZE_M": 1,
62
- "num_warps": 8,
62
+ "num_warps": 1,
63
"num_stages": 2,
64
"waves_per_eu": 0,
65
"matrix_instr_nonkdim": 16,
@@ -68,7 +68,7 @@
68
"32": {
69
"BLOCK_SIZE_M": 16,
70
"BLOCK_SIZE_N": 32,
71
- "BLOCK_SIZE_K": 128,
71
+ "BLOCK_SIZE_K": 256,
72
"GROUP_SIZE_M": 4,
73
"num_warps": 2,
74
"num_stages": 2,
@@ -78,32 +78,32 @@
78
},
79
"48": {
80
"BLOCK_SIZE_M": 16,
81
- "BLOCK_SIZE_N": 32,
82
- "BLOCK_SIZE_K": 128,
83
- "GROUP_SIZE_M": 4,
81
+ "BLOCK_SIZE_N": 16,
82
+ "BLOCK_SIZE_K": 256,
83
+ "GROUP_SIZE_M": 1,
84
"num_warps": 2,
85
"num_stages": 2,
86
"waves_per_eu": 0,
87
"matrix_instr_nonkdim": 16,
88
- "kpack": 1
88
+ "kpack": 2
89
},
90
"64": {
91
"BLOCK_SIZE_M": 32,
92
- "BLOCK_SIZE_N": 32,
92
+ "BLOCK_SIZE_N": 64,
93
"BLOCK_SIZE_K": 128,
94
"GROUP_SIZE_M": 4,
95
"num_warps": 4,
96
"num_stages": 2,
97
"waves_per_eu": 0,
98
"matrix_instr_nonkdim": 16,
99
- "kpack": 2
99
+ "kpack": 1
100
},
101
"96": {
102
"BLOCK_SIZE_M": 32,
103
- "BLOCK_SIZE_N": 32,
104
- "BLOCK_SIZE_K": 128,
103
+ "BLOCK_SIZE_N": 64,
104
+ "BLOCK_SIZE_K": 256,
105
"GROUP_SIZE_M": 4,
106
- "num_warps": 4,
106
+ "num_warps": 8,
107
"num_stages": 2,
108
"waves_per_eu": 0,
109
"matrix_instr_nonkdim": 16,
@@ -112,18 +112,18 @@
112
"128": {
113
"BLOCK_SIZE_M": 64,
114
"BLOCK_SIZE_N": 64,
115
- "BLOCK_SIZE_K": 64,
115
+ "BLOCK_SIZE_K": 128,
116
"GROUP_SIZE_M": 4,
117
"num_warps": 8,
118
"num_stages": 2,
119
"waves_per_eu": 0,
120
"matrix_instr_nonkdim": 16,
121
- "kpack": 1
121
+ "kpack": 2
122
},
123
"256": {
124
"BLOCK_SIZE_M": 128,
125
"BLOCK_SIZE_N": 128,
126
- "BLOCK_SIZE_K": 64,
126
+ "BLOCK_SIZE_K": 128,
127
"GROUP_SIZE_M": 4,
128
"num_warps": 8,
129
"num_stages": 2,
@@ -140,7 +140,7 @@
140
"num_stages": 2,
141
"waves_per_eu": 0,
142
"matrix_instr_nonkdim": 16,
143
- "kpack": 1
143
+ "kpack": 2
144
},
145
"1024": {
146
"BLOCK_SIZE_M": 128,
@@ -151,7 +151,7 @@
151
"num_stages": 2,
152
"waves_per_eu": 0,
153
"matrix_instr_nonkdim": 16,
154
- "kpack": 1
154
+ "kpack": 2
155
},
156
"1536": {
157
"BLOCK_SIZE_M": 128,
@@ -173,7 +173,7 @@
173
"num_stages": 2,
174
"waves_per_eu": 0,
175
"matrix_instr_nonkdim": 16,
176
- "kpack": 1
176
+ "kpack": 2
177
},
178
"3072": {
179
"BLOCK_SIZE_M": 128,
@@ -187,7 +187,7 @@
187
"kpack": 2
188
},
189
"4096": {
190
- "BLOCK_SIZE_M": 128,
190
+ "BLOCK_SIZE_M": 256,
191
"BLOCK_SIZE_N": 128,
192
"BLOCK_SIZE_K": 64,
193
"GROUP_SIZE_M": 1,
@@ -195,6 +195,6 @@
195
"num_stages": 2,
196
"waves_per_eu": 0,
197
"matrix_instr_nonkdim": 16,
198
- "kpack": 1
198
+ "kpack": 2
199
}
200
}
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json
new
+164
@@ -0,0 +1,164 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 32,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 2,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0
10
+ },
11
+ "2": {
12
+ "BLOCK_SIZE_M": 16,
13
+ "BLOCK_SIZE_N": 128,
14
+ "BLOCK_SIZE_K": 256,
15
+ "GROUP_SIZE_M": 1,
16
+ "num_warps": 4,
17
+ "num_stages": 2,
18
+ "waves_per_eu": 0
19
+ },
20
+ "4": {
21
+ "BLOCK_SIZE_M": 16,
22
+ "BLOCK_SIZE_N": 32,
23
+ "BLOCK_SIZE_K": 128,
24
+ "GROUP_SIZE_M": 1,
25
+ "num_warps": 2,
26
+ "num_stages": 2,
27
+ "waves_per_eu": 0
28
+ },
29
+ "8": {
30
+ "BLOCK_SIZE_M": 16,
31
+ "BLOCK_SIZE_N": 64,
32
+ "BLOCK_SIZE_K": 256,
33
+ "GROUP_SIZE_M": 1,
34
+ "num_warps": 2,
35
+ "num_stages": 2,
36
+ "waves_per_eu": 0
37
+ },
38
+ "16": {
39
+ "BLOCK_SIZE_M": 16,
40
+ "BLOCK_SIZE_N": 64,
41
+ "BLOCK_SIZE_K": 256,
42
+ "GROUP_SIZE_M": 1,
43
+ "num_warps": 1,
44
+ "num_stages": 2,
45
+ "waves_per_eu": 0
46
+ },
47
+ "24": {
48
+ "BLOCK_SIZE_M": 16,
49
+ "BLOCK_SIZE_N": 64,
50
+ "BLOCK_SIZE_K": 256,
51
+ "GROUP_SIZE_M": 1,
52
+ "num_warps": 2,
53
+ "num_stages": 2,
54
+ "waves_per_eu": 0
55
+ },
56
+ "32": {
57
+ "BLOCK_SIZE_M": 16,
58
+ "BLOCK_SIZE_N": 64,
59
+ "BLOCK_SIZE_K": 256,
60
+ "GROUP_SIZE_M": 4,
61
+ "num_warps": 4,
62
+ "num_stages": 2,
63
+ "waves_per_eu": 0
64
+ },
65
+ "48": {
66
+ "BLOCK_SIZE_M": 16,
67
+ "BLOCK_SIZE_N": 64,
68
+ "BLOCK_SIZE_K": 256,
69
+ "GROUP_SIZE_M": 1,
70
+ "num_warps": 2,
71
+ "num_stages": 2,
72
+ "waves_per_eu": 0
73
+ },
74
+ "64": {
75
+ "BLOCK_SIZE_M": 32,
76
+ "BLOCK_SIZE_N": 32,
77
+ "BLOCK_SIZE_K": 128,
78
+ "GROUP_SIZE_M": 4,
79
+ "num_warps": 1,
80
+ "num_stages": 2,
81
+ "waves_per_eu": 0
82
+ },
83
+ "96": {
84
+ "BLOCK_SIZE_M": 64,
85
+ "BLOCK_SIZE_N": 64,
86
+ "BLOCK_SIZE_K": 128,
87
+ "GROUP_SIZE_M": 1,
88
+ "num_warps": 4,
89
+ "num_stages": 2,
90
+ "waves_per_eu": 0
91
+ },
92
+ "128": {
93
+ "BLOCK_SIZE_M": 64,
94
+ "BLOCK_SIZE_N": 64,
95
+ "BLOCK_SIZE_K": 128,
96
+ "GROUP_SIZE_M": 4,
97
+ "num_warps": 2,
98
+ "num_stages": 2,
99
+ "waves_per_eu": 0
100
+ },
101
+ "256": {
102
+ "BLOCK_SIZE_M": 64,
103
+ "BLOCK_SIZE_N": 64,
104
+ "BLOCK_SIZE_K": 128,
105
+ "GROUP_SIZE_M": 1,
106
+ "num_warps": 4,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0
109
+ },
110
+ "512": {
111
+ "BLOCK_SIZE_M": 128,
112
+ "BLOCK_SIZE_N": 128,
113
+ "BLOCK_SIZE_K": 256,
114
+ "GROUP_SIZE_M": 1,
115
+ "num_warps": 8,
116
+ "num_stages": 2,
117
+ "waves_per_eu": 0
118
+ },
119
+ "1024": {
120
+ "BLOCK_SIZE_M": 128,
121
+ "BLOCK_SIZE_N": 256,
122
+ "BLOCK_SIZE_K": 128,
123
+ "GROUP_SIZE_M": 1,
124
+ "num_warps": 8,
125
+ "num_stages": 2,
126
+ "waves_per_eu": 0
127
+ },
128
+ "1536": {
129
+ "BLOCK_SIZE_M": 128,
130
+ "BLOCK_SIZE_N": 256,
131
+ "BLOCK_SIZE_K": 128,
132
+ "GROUP_SIZE_M": 1,
133
+ "num_warps": 8,
134
+ "num_stages": 2,
135
+ "waves_per_eu": 0
136
+ },
137
+ "2048": {
138
+ "BLOCK_SIZE_M": 128,
139
+ "BLOCK_SIZE_N": 256,
140
+ "BLOCK_SIZE_K": 128,
141
+ "GROUP_SIZE_M": 1,
142
+ "num_warps": 8,
143
+ "num_stages": 2,
144
+ "waves_per_eu": 0
145
+ },
146
+ "3072": {
147
+ "BLOCK_SIZE_M": 128,
148
+ "BLOCK_SIZE_N": 256,
149
+ "BLOCK_SIZE_K": 128,
150
+ "GROUP_SIZE_M": 1,
151
+ "num_warps": 8,
152
+ "num_stages": 2,
153
+ "waves_per_eu": 0
154
+ },
155
+ "4096": {
156
+ "BLOCK_SIZE_M": 256,
157
+ "BLOCK_SIZE_N": 256,
158
+ "BLOCK_SIZE_K": 64,
159
+ "GROUP_SIZE_M": 1,
160
+ "num_warps": 8,
161
+ "num_stages": 2,
162
+ "waves_per_eu": 0
163
+ }
164
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X.json
new
+200
@@ -0,0 +1,200 @@
1
+{
2
+ "1": {
3
+ "BLOCK_SIZE_M": 16,
4
+ "BLOCK_SIZE_N": 16,
5
+ "BLOCK_SIZE_K": 256,
6
+ "GROUP_SIZE_M": 1,
7
+ "num_warps": 1,
8
+ "num_stages": 2,
9
+ "waves_per_eu": 0,
10
+ "matrix_instr_nonkdim": 16,
11
+ "kpack": 1
12
+ },
13
+ "2": {
14
+ "BLOCK_SIZE_M": 16,
15
+ "BLOCK_SIZE_N": 64,
16
+ "BLOCK_SIZE_K": 128,
17
+ "GROUP_SIZE_M": 1,
18
+ "num_warps": 2,
19
+ "num_stages": 2,
20
+ "waves_per_eu": 0,
21
+ "matrix_instr_nonkdim": 16,
22
+ "kpack": 2
23
+ },
24
+ "4": {
25
+ "BLOCK_SIZE_M": 16,
26
+ "BLOCK_SIZE_N": 64,
27
+ "BLOCK_SIZE_K": 128,
28
+ "GROUP_SIZE_M": 1,
29
+ "num_warps": 4,
30
+ "num_stages": 2,
31
+ "waves_per_eu": 0,
32
+ "matrix_instr_nonkdim": 16,
33
+ "kpack": 2
34
+ },
35
+ "8": {
36
+ "BLOCK_SIZE_M": 16,
37
+ "BLOCK_SIZE_N": 16,
38
+ "BLOCK_SIZE_K": 256,
39
+ "GROUP_SIZE_M": 1,
40
+ "num_warps": 1,
41
+ "num_stages": 2,
42
+ "waves_per_eu": 0,
43
+ "matrix_instr_nonkdim": 16,
44
+ "kpack": 1
45
+ },
46
+ "16": {
47
+ "BLOCK_SIZE_M": 16,
48
+ "BLOCK_SIZE_N": 32,
49
+ "BLOCK_SIZE_K": 64,
50
+ "GROUP_SIZE_M": 1,
51
+ "num_warps": 2,
52
+ "num_stages": 2,
53
+ "waves_per_eu": 0,
54
+ "matrix_instr_nonkdim": 16,
55
+ "kpack": 2
56
+ },
57
+ "24": {
58
+ "BLOCK_SIZE_M": 16,
59
+ "BLOCK_SIZE_N": 64,
60
+ "BLOCK_SIZE_K": 128,
61
+ "GROUP_SIZE_M": 1,
62
+ "num_warps": 2,
63
+ "num_stages": 2,
64
+ "waves_per_eu": 0,
65
+ "matrix_instr_nonkdim": 16,
66
+ "kpack": 2
67
+ },
68
+ "32": {
69
+ "BLOCK_SIZE_M": 16,
70
+ "BLOCK_SIZE_N": 64,
71
+ "BLOCK_SIZE_K": 128,
72
+ "GROUP_SIZE_M": 4,
73
+ "num_warps": 2,
74
+ "num_stages": 2,
75
+ "waves_per_eu": 0,
76
+ "matrix_instr_nonkdim": 16,
77
+ "kpack": 2
78
+ },
79
+ "48": {
80
+ "BLOCK_SIZE_M": 32,
81
+ "BLOCK_SIZE_N": 64,
82
+ "BLOCK_SIZE_K": 64,
83
+ "GROUP_SIZE_M": 1,
84
+ "num_warps": 8,
85
+ "num_stages": 2,
86
+ "waves_per_eu": 0,
87
+ "matrix_instr_nonkdim": 16,
88
+ "kpack": 2
89
+ },
90
+ "64": {
91
+ "BLOCK_SIZE_M": 32,
92
+ "BLOCK_SIZE_N": 32,
93
+ "BLOCK_SIZE_K": 256,
94
+ "GROUP_SIZE_M": 4,
95
+ "num_warps": 4,
96
+ "num_stages": 2,
97
+ "waves_per_eu": 0,
98
+ "matrix_instr_nonkdim": 16,
99
+ "kpack": 2
100
+ },
101
+ "96": {
102
+ "BLOCK_SIZE_M": 32,
103
+ "BLOCK_SIZE_N": 32,
104
+ "BLOCK_SIZE_K": 256,
105
+ "GROUP_SIZE_M": 4,
106
+ "num_warps": 4,
107
+ "num_stages": 2,
108
+ "waves_per_eu": 0,
109
+ "matrix_instr_nonkdim": 16,
110
+ "kpack": 2
111
+ },
112
+ "128": {
113
+ "BLOCK_SIZE_M": 64,
114
+ "BLOCK_SIZE_N": 64,
115
+ "BLOCK_SIZE_K": 64,
116
+ "GROUP_SIZE_M": 4,
117
+ "num_warps": 4,
118
+ "num_stages": 2,
119
+ "waves_per_eu": 0,
120
+ "matrix_instr_nonkdim": 16,
121
+ "kpack": 2
122
+ },
123
+ "256": {
124
+ "BLOCK_SIZE_M": 128,
125
+ "BLOCK_SIZE_N": 128,
126
+ "BLOCK_SIZE_K": 64,
127
+ "GROUP_SIZE_M": 1,
128
+ "num_warps": 4,
129
+ "num_stages": 2,
130
+ "waves_per_eu": 0,
131
+ "matrix_instr_nonkdim": 16,
132
+ "kpack": 2
133
+ },
134
+ "512": {
135
+ "BLOCK_SIZE_M": 128,
136
+ "BLOCK_SIZE_N": 128,
137
+ "BLOCK_SIZE_K": 64,
138
+ "GROUP_SIZE_M": 1,
139
+ "num_warps": 8,
140
+ "num_stages": 2,
141
+ "waves_per_eu": 0,
142
+ "matrix_instr_nonkdim": 16,
143
+ "kpack": 2
144
+ },
145
+ "1024": {
146
+ "BLOCK_SIZE_M": 128,
147
+ "BLOCK_SIZE_N": 128,
148
+ "BLOCK_SIZE_K": 64,
149
+ "GROUP_SIZE_M": 1,
150
+ "num_warps": 8,
151
+ "num_stages": 2,
152
+ "waves_per_eu": 0,
153
+ "matrix_instr_nonkdim": 16,
154
+ "kpack": 2
155
+ },
156
+ "1536": {
157
+ "BLOCK_SIZE_M": 128,
158
+ "BLOCK_SIZE_N": 128,
159
+ "BLOCK_SIZE_K": 64,
160
+ "GROUP_SIZE_M": 1,
161
+ "num_warps": 8,
162
+ "num_stages": 2,
163
+ "waves_per_eu": 0,
164
+ "matrix_instr_nonkdim": 16,
165
+ "kpack": 2
166
+ },
167
+ "2048": {
168
+ "BLOCK_SIZE_M": 128,
169
+ "BLOCK_SIZE_N": 128,
170
+ "BLOCK_SIZE_K": 64,
171
+ "GROUP_SIZE_M": 1,
172
+ "num_warps": 8,
173
+ "num_stages": 2,
174
+ "waves_per_eu": 0,
175
+ "matrix_instr_nonkdim": 16,
176
+ "kpack": 2
177
+ },
178
+ "3072": {
179
+ "BLOCK_SIZE_M": 128,
180
+ "BLOCK_SIZE_N": 128,
181
+ "BLOCK_SIZE_K": 64,
182
+ "GROUP_SIZE_M": 1,
183
+ "num_warps": 8,
184
+ "num_stages": 2,
185
+ "waves_per_eu": 0,
186
+ "matrix_instr_nonkdim": 16,
187
+ "kpack": 2
188
+ },
189
+ "4096": {
190
+ "BLOCK_SIZE_M": 128,
191
+ "BLOCK_SIZE_N": 128,
192
+ "BLOCK_SIZE_K": 64,
193
+ "GROUP_SIZE_M": 1,
194
+ "num_warps": 8,
195
+ "num_stages": 2,
196
+ "waves_per_eu": 0,
197
+ "matrix_instr_nonkdim": 16,
198
+ "kpack": 2
199
+ }
200
+}