@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf21481dde
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[ROCm][MoE] MI300 tuned configs Mixtral-8x(7B,22B) | fp16, fp8 (#12408)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>

Divakar Verma committed Jan 24, 2025 at 22:17 UTC bf21481ddef2fa9bb96c13ba1f80072abdae3eb7
16 files changed +2260 -148
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 16,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 2,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 256,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 4,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 32,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 2,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 256,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 2,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 64,
50 + "BLOCK_SIZE_K": 256,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 2,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 32,
59 + "BLOCK_SIZE_K": 256,
60 + "GROUP_SIZE_M": 4,
61 + "num_warps": 2,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 64,
68 + "BLOCK_SIZE_K": 128,
69 + "GROUP_SIZE_M": 4,
70 + "num_warps": 4,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 4,
79 + "num_warps": 2,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 32,
85 + "BLOCK_SIZE_N": 64,
86 + "BLOCK_SIZE_K": 256,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 2,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 128,
95 + "BLOCK_SIZE_K": 256,
96 + "GROUP_SIZE_M": 4,
97 + "num_warps": 8,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 128,
103 + "BLOCK_SIZE_N": 128,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 8,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 256,
112 + "BLOCK_SIZE_N": 128,
113 + "BLOCK_SIZE_K": 128,
114 + "GROUP_SIZE_M": 4,
115 + "num_warps": 8,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 128,
121 + "BLOCK_SIZE_N": 128,
122 + "BLOCK_SIZE_K": 64,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 4,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 128,
130 + "BLOCK_SIZE_N": 256,
131 + "BLOCK_SIZE_K": 128,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 8,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 256,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 8,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 128,
150 + "GROUP_SIZE_M": 1,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 256,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 64,
159 + "GROUP_SIZE_M": 1,
160 + "num_warps": 8,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json
+32 -32
@@ -1,21 +1,21 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 32,
4 + "BLOCK_SIZE_N": 16,
5 "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 "num_warps": 2,
8 "num_stages": 2,
9 "waves_per_eu": 0,
10 "matrix_instr_nonkdim": 16,
11 - "kpack": 1
11 + "kpack": 2
12 },
13 "2": {
14 "BLOCK_SIZE_M": 16,
15 "BLOCK_SIZE_N": 16,
16 - "BLOCK_SIZE_K": 128,
16 + "BLOCK_SIZE_K": 256,
17 "GROUP_SIZE_M": 1,
18 - "num_warps": 2,
18 + "num_warps": 4,
19 "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
@@ -23,10 +23,10 @@
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
26 - "BLOCK_SIZE_N": 32,
27 - "BLOCK_SIZE_K": 256,
26 + "BLOCK_SIZE_N": 16,
27 + "BLOCK_SIZE_K": 128,
28 "GROUP_SIZE_M": 1,
29 - "num_warps": 2,
29 + "num_warps": 1,
30 "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
@@ -34,10 +34,10 @@
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
37 - "BLOCK_SIZE_N": 16,
38 - "BLOCK_SIZE_K": 256,
37 + "BLOCK_SIZE_N": 64,
38 + "BLOCK_SIZE_K": 64,
39 "GROUP_SIZE_M": 1,
40 - "num_warps": 1,
40 + "num_warps": 2,
41 "num_stages": 2,
42 "waves_per_eu": 0,
43 "matrix_instr_nonkdim": 16,
@@ -48,7 +48,7 @@
48 "BLOCK_SIZE_N": 16,
49 "BLOCK_SIZE_K": 256,
50 "GROUP_SIZE_M": 1,
51 - "num_warps": 4,
51 + "num_warps": 2,
52 "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
@@ -56,10 +56,10 @@
56 },
57 "24": {
58 "BLOCK_SIZE_M": 16,
59 - "BLOCK_SIZE_N": 32,
60 - "BLOCK_SIZE_K": 64,
59 + "BLOCK_SIZE_N": 16,
60 + "BLOCK_SIZE_K": 256,
61 "GROUP_SIZE_M": 1,
62 - "num_warps": 1,
62 + "num_warps": 2,
63 "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
@@ -67,32 +67,32 @@
67 },
68 "32": {
69 "BLOCK_SIZE_M": 16,
70 - "BLOCK_SIZE_N": 16,
71 - "BLOCK_SIZE_K": 128,
70 + "BLOCK_SIZE_N": 32,
71 + "BLOCK_SIZE_K": 256,
72 "GROUP_SIZE_M": 4,
73 "num_warps": 2,
74 "num_stages": 2,
75 "waves_per_eu": 0,
76 "matrix_instr_nonkdim": 16,
77 - "kpack": 1
77 + "kpack": 2
78 },
79 "48": {
80 "BLOCK_SIZE_M": 16,
81 - "BLOCK_SIZE_N": 16,
81 + "BLOCK_SIZE_N": 64,
82 "BLOCK_SIZE_K": 128,
83 "GROUP_SIZE_M": 4,
84 - "num_warps": 2,
84 + "num_warps": 4,
85 "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 - "kpack": 2
88 + "kpack": 1
89 },
90 "64": {
91 "BLOCK_SIZE_M": 32,
92 "BLOCK_SIZE_N": 64,
93 "BLOCK_SIZE_K": 128,
94 "GROUP_SIZE_M": 4,
95 - "num_warps": 8,
95 + "num_warps": 4,
96 "num_stages": 2,
97 "waves_per_eu": 0,
98 "matrix_instr_nonkdim": 16,
@@ -100,24 +100,24 @@
100 },
101 "96": {
102 "BLOCK_SIZE_M": 32,
103 - "BLOCK_SIZE_N": 32,
104 - "BLOCK_SIZE_K": 128,
103 + "BLOCK_SIZE_N": 64,
104 + "BLOCK_SIZE_K": 256,
105 "GROUP_SIZE_M": 4,
106 - "num_warps": 4,
106 + "num_warps": 8,
107 "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
110 - "kpack": 2
110 + "kpack": 1
111 },
112 "128": {
113 "BLOCK_SIZE_M": 64,
114 "BLOCK_SIZE_N": 64,
115 - "BLOCK_SIZE_K": 64,
115 + "BLOCK_SIZE_K": 128,
116 "GROUP_SIZE_M": 4,
117 - "num_warps": 8,
117 + "num_warps": 4,
118 "num_stages": 2,
119 "waves_per_eu": 0,
120 - "matrix_instr_nonkdim": 16,
120 + "matrix_instr_nonkdim": 32,
121 "kpack": 2
122 },
123 "256": {
@@ -129,7 +129,7 @@
129 "num_stages": 2,
130 "waves_per_eu": 0,
131 "matrix_instr_nonkdim": 16,
132 - "kpack": 1
132 + "kpack": 2
133 },
134 "512": {
135 "BLOCK_SIZE_M": 128,
@@ -150,7 +150,7 @@
150 "num_warps": 8,
151 "num_stages": 2,
152 "waves_per_eu": 0,
153 - "matrix_instr_nonkdim": 32,
153 + "matrix_instr_nonkdim": 16,
154 "kpack": 2
155 },
156 "1536": {
@@ -184,7 +184,7 @@
184 "num_stages": 2,
185 "waves_per_eu": 0,
186 "matrix_instr_nonkdim": 16,
187 - "kpack": 1
187 + "kpack": 2
188 },
189 "4096": {
190 "BLOCK_SIZE_M": 128,
@@ -195,6 +195,6 @@
195 "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 - "kpack": 1
198 + "kpack": 2
199 }
200 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 16,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 2,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 32,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 2,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 256,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 2,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 64,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 2,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 256,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 2,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 64,
50 + "BLOCK_SIZE_K": 256,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 2,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 64,
59 + "BLOCK_SIZE_K": 256,
60 + "GROUP_SIZE_M": 4,
61 + "num_warps": 2,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 64,
68 + "BLOCK_SIZE_K": 256,
69 + "GROUP_SIZE_M": 1,
70 + "num_warps": 4,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 4,
79 + "num_warps": 2,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 32,
85 + "BLOCK_SIZE_N": 64,
86 + "BLOCK_SIZE_K": 128,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 2,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 64,
95 + "BLOCK_SIZE_K": 128,
96 + "GROUP_SIZE_M": 4,
97 + "num_warps": 4,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 64,
103 + "BLOCK_SIZE_N": 128,
104 + "BLOCK_SIZE_K": 128,
105 + "GROUP_SIZE_M": 1,
106 + "num_warps": 8,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 128,
112 + "BLOCK_SIZE_N": 128,
113 + "BLOCK_SIZE_K": 256,
114 + "GROUP_SIZE_M": 1,
115 + "num_warps": 8,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 128,
121 + "BLOCK_SIZE_N": 256,
122 + "BLOCK_SIZE_K": 128,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 8,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 128,
130 + "BLOCK_SIZE_N": 256,
131 + "BLOCK_SIZE_K": 128,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 8,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 256,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 8,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 128,
150 + "GROUP_SIZE_M": 1,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 256,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 64,
159 + "GROUP_SIZE_M": 1,
160 + "num_warps": 8,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=16384,device_name=AMD_Instinct_MI300X.json new
+200
@@ -0,0 +1,200 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 16,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 2,
8 + "num_stages": 2,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 2
12 + },
13 + "2": {
14 + "BLOCK_SIZE_M": 16,
15 + "BLOCK_SIZE_N": 64,
16 + "BLOCK_SIZE_K": 128,
17 + "GROUP_SIZE_M": 1,
18 + "num_warps": 2,
19 + "num_stages": 2,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 2
23 + },
24 + "4": {
25 + "BLOCK_SIZE_M": 16,
26 + "BLOCK_SIZE_N": 64,
27 + "BLOCK_SIZE_K": 64,
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 2,
30 + "num_stages": 2,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 2
34 + },
35 + "8": {
36 + "BLOCK_SIZE_M": 16,
37 + "BLOCK_SIZE_N": 32,
38 + "BLOCK_SIZE_K": 64,
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 2,
41 + "num_stages": 2,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 2
45 + },
46 + "16": {
47 + "BLOCK_SIZE_M": 16,
48 + "BLOCK_SIZE_N": 16,
49 + "BLOCK_SIZE_K": 256,
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 1,
52 + "num_stages": 2,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 2
56 + },
57 + "24": {
58 + "BLOCK_SIZE_M": 16,
59 + "BLOCK_SIZE_N": 64,
60 + "BLOCK_SIZE_K": 128,
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 2,
63 + "num_stages": 2,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 2
67 + },
68 + "32": {
69 + "BLOCK_SIZE_M": 16,
70 + "BLOCK_SIZE_N": 64,
71 + "BLOCK_SIZE_K": 128,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 2,
74 + "num_stages": 2,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 2
78 + },
79 + "48": {
80 + "BLOCK_SIZE_M": 32,
81 + "BLOCK_SIZE_N": 32,
82 + "BLOCK_SIZE_K": 256,
83 + "GROUP_SIZE_M": 1,
84 + "num_warps": 4,
85 + "num_stages": 2,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 2
89 + },
90 + "64": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 256,
94 + "GROUP_SIZE_M": 4,
95 + "num_warps": 4,
96 + "num_stages": 2,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 2
100 + },
101 + "96": {
102 + "BLOCK_SIZE_M": 32,
103 + "BLOCK_SIZE_N": 32,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 2,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 1
111 + },
112 + "128": {
113 + "BLOCK_SIZE_M": 64,
114 + "BLOCK_SIZE_N": 64,
115 + "BLOCK_SIZE_K": 128,
116 + "GROUP_SIZE_M": 4,
117 + "num_warps": 8,
118 + "num_stages": 2,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 2
122 + },
123 + "256": {
124 + "BLOCK_SIZE_M": 128,
125 + "BLOCK_SIZE_N": 128,
126 + "BLOCK_SIZE_K": 64,
127 + "GROUP_SIZE_M": 4,
128 + "num_warps": 8,
129 + "num_stages": 2,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 16,
132 + "kpack": 2
133 + },
134 + "512": {
135 + "BLOCK_SIZE_M": 128,
136 + "BLOCK_SIZE_N": 128,
137 + "BLOCK_SIZE_K": 64,
138 + "GROUP_SIZE_M": 1,
139 + "num_warps": 8,
140 + "num_stages": 2,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 16,
143 + "kpack": 2
144 + },
145 + "1024": {
146 + "BLOCK_SIZE_M": 128,
147 + "BLOCK_SIZE_N": 128,
148 + "BLOCK_SIZE_K": 64,
149 + "GROUP_SIZE_M": 4,
150 + "num_warps": 8,
151 + "num_stages": 2,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 16,
154 + "kpack": 2
155 + },
156 + "1536": {
157 + "BLOCK_SIZE_M": 128,
158 + "BLOCK_SIZE_N": 128,
159 + "BLOCK_SIZE_K": 64,
160 + "GROUP_SIZE_M": 4,
161 + "num_warps": 8,
162 + "num_stages": 2,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 2
166 + },
167 + "2048": {
168 + "BLOCK_SIZE_M": 128,
169 + "BLOCK_SIZE_N": 128,
170 + "BLOCK_SIZE_K": 64,
171 + "GROUP_SIZE_M": 4,
172 + "num_warps": 8,
173 + "num_stages": 2,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 2
177 + },
178 + "3072": {
179 + "BLOCK_SIZE_M": 128,
180 + "BLOCK_SIZE_N": 128,
181 + "BLOCK_SIZE_K": 64,
182 + "GROUP_SIZE_M": 16,
183 + "num_warps": 8,
184 + "num_stages": 2,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 2
188 + },
189 + "4096": {
190 + "BLOCK_SIZE_M": 128,
191 + "BLOCK_SIZE_N": 128,
192 + "BLOCK_SIZE_K": 64,
193 + "GROUP_SIZE_M": 16,
194 + "num_warps": 8,
195 + "num_stages": 2,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 2
199 + }
200 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 64,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 4,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 128,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 4,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 128,
32 + "BLOCK_SIZE_K": 128,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 8,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 128,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 4,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 128,
50 + "BLOCK_SIZE_K": 128,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 8,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 128,
59 + "BLOCK_SIZE_K": 128,
60 + "GROUP_SIZE_M": 4,
61 + "num_warps": 8,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 128,
68 + "BLOCK_SIZE_K": 128,
69 + "GROUP_SIZE_M": 1,
70 + "num_warps": 8,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 4,
79 + "num_warps": 4,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 32,
85 + "BLOCK_SIZE_N": 128,
86 + "BLOCK_SIZE_K": 128,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 4,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 128,
95 + "BLOCK_SIZE_K": 128,
96 + "GROUP_SIZE_M": 4,
97 + "num_warps": 8,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 128,
103 + "BLOCK_SIZE_N": 128,
104 + "BLOCK_SIZE_K": 128,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 8,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 64,
112 + "BLOCK_SIZE_N": 128,
113 + "BLOCK_SIZE_K": 128,
114 + "GROUP_SIZE_M": 1,
115 + "num_warps": 4,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 128,
121 + "BLOCK_SIZE_N": 256,
122 + "BLOCK_SIZE_K": 128,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 8,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 64,
130 + "BLOCK_SIZE_N": 128,
131 + "BLOCK_SIZE_K": 128,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 4,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 256,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 8,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 128,
149 + "BLOCK_SIZE_K": 256,
150 + "GROUP_SIZE_M": 1,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 128,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 128,
159 + "GROUP_SIZE_M": 4,
160 + "num_warps": 8,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json
+43 -43
@@ -1,10 +1,10 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 32,
5 - "BLOCK_SIZE_K": 256,
4 + "BLOCK_SIZE_N": 16,
5 + "BLOCK_SIZE_K": 128,
6 "GROUP_SIZE_M": 1,
7 - "num_warps": 2,
7 + "num_warps": 4,
8 "num_stages": 2,
9 "waves_per_eu": 0,
10 "matrix_instr_nonkdim": 16,
@@ -19,14 +19,14 @@
19 "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
22 - "kpack": 1
22 + "kpack": 2
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
26 - "BLOCK_SIZE_N": 64,
26 + "BLOCK_SIZE_N": 128,
27 "BLOCK_SIZE_K": 128,
28 "GROUP_SIZE_M": 1,
29 - "num_warps": 4,
29 + "num_warps": 8,
30 "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
@@ -34,76 +34,76 @@
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
37 - "BLOCK_SIZE_N": 16,
38 - "BLOCK_SIZE_K": 256,
37 + "BLOCK_SIZE_N": 64,
38 + "BLOCK_SIZE_K": 128,
39 "GROUP_SIZE_M": 1,
40 - "num_warps": 2,
40 + "num_warps": 4,
41 "num_stages": 2,
42 "waves_per_eu": 0,
43 "matrix_instr_nonkdim": 16,
44 - "kpack": 2
44 + "kpack": 1
45 },
46 "16": {
47 - "BLOCK_SIZE_M": 64,
47 + "BLOCK_SIZE_M": 16,
48 "BLOCK_SIZE_N": 64,
49 - "BLOCK_SIZE_K": 64,
49 + "BLOCK_SIZE_K": 128,
50 "GROUP_SIZE_M": 1,
51 - "num_warps": 8,
51 + "num_warps": 4,
52 "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
55 - "kpack": 1
55 + "kpack": 2
56 },
57 "24": {
58 "BLOCK_SIZE_M": 16,
59 "BLOCK_SIZE_N": 64,
60 - "BLOCK_SIZE_K": 64,
60 + "BLOCK_SIZE_K": 128,
61 "GROUP_SIZE_M": 1,
62 "num_warps": 4,
63 "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
66 - "kpack": 1
66 + "kpack": 2
67 },
68 "32": {
69 "BLOCK_SIZE_M": 16,
70 - "BLOCK_SIZE_N": 16,
71 - "BLOCK_SIZE_K": 256,
70 + "BLOCK_SIZE_N": 64,
71 + "BLOCK_SIZE_K": 128,
72 "GROUP_SIZE_M": 4,
73 - "num_warps": 2,
73 + "num_warps": 4,
74 "num_stages": 2,
75 "waves_per_eu": 0,
76 "matrix_instr_nonkdim": 16,
77 - "kpack": 2
77 + "kpack": 1
78 },
79 "48": {
80 "BLOCK_SIZE_M": 16,
81 - "BLOCK_SIZE_N": 128,
82 - "BLOCK_SIZE_K": 64,
83 - "GROUP_SIZE_M": 1,
84 - "num_warps": 8,
81 + "BLOCK_SIZE_N": 64,
82 + "BLOCK_SIZE_K": 128,
83 + "GROUP_SIZE_M": 4,
84 + "num_warps": 2,
85 "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 "kpack": 2
89 },
90 "64": {
91 - "BLOCK_SIZE_M": 16,
91 + "BLOCK_SIZE_M": 32,
92 "BLOCK_SIZE_N": 64,
93 - "BLOCK_SIZE_K": 64,
93 + "BLOCK_SIZE_K": 128,
94 "GROUP_SIZE_M": 1,
95 - "num_warps": 2,
95 + "num_warps": 4,
96 "num_stages": 2,
97 "waves_per_eu": 0,
98 "matrix_instr_nonkdim": 16,
99 - "kpack": 1
99 + "kpack": 2
100 },
101 "96": {
102 "BLOCK_SIZE_M": 32,
103 - "BLOCK_SIZE_N": 32,
104 - "BLOCK_SIZE_K": 256,
105 - "GROUP_SIZE_M": 4,
106 - "num_warps": 4,
103 + "BLOCK_SIZE_N": 128,
104 + "BLOCK_SIZE_K": 128,
105 + "GROUP_SIZE_M": 1,
106 + "num_warps": 8,
107 "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
@@ -112,24 +112,24 @@
112 "128": {
113 "BLOCK_SIZE_M": 64,
114 "BLOCK_SIZE_N": 64,
115 - "BLOCK_SIZE_K": 64,
115 + "BLOCK_SIZE_K": 128,
116 "GROUP_SIZE_M": 4,
117 "num_warps": 8,
118 "num_stages": 2,
119 "waves_per_eu": 0,
120 "matrix_instr_nonkdim": 16,
121 - "kpack": 1
121 + "kpack": 2
122 },
123 "256": {
124 - "BLOCK_SIZE_M": 128,
125 - "BLOCK_SIZE_N": 128,
126 - "BLOCK_SIZE_K": 64,
127 - "GROUP_SIZE_M": 4,
124 + "BLOCK_SIZE_M": 32,
125 + "BLOCK_SIZE_N": 64,
126 + "BLOCK_SIZE_K": 128,
127 + "GROUP_SIZE_M": 1,
128 "num_warps": 8,
129 "num_stages": 2,
130 "waves_per_eu": 0,
131 "matrix_instr_nonkdim": 16,
132 - "kpack": 1
132 + "kpack": 2
133 },
134 "512": {
135 "BLOCK_SIZE_M": 64,
@@ -151,7 +151,7 @@
151 "num_stages": 2,
152 "waves_per_eu": 0,
153 "matrix_instr_nonkdim": 16,
154 - "kpack": 1
154 + "kpack": 2
155 },
156 "1536": {
157 "BLOCK_SIZE_M": 128,
@@ -162,7 +162,7 @@
162 "num_stages": 2,
163 "waves_per_eu": 0,
164 "matrix_instr_nonkdim": 16,
165 - "kpack": 1
165 + "kpack": 2
166 },
167 "2048": {
168 "BLOCK_SIZE_M": 128,
@@ -184,7 +184,7 @@
184 "num_stages": 2,
185 "waves_per_eu": 0,
186 "matrix_instr_nonkdim": 16,
187 - "kpack": 1
187 + "kpack": 2
188 },
189 "4096": {
190 "BLOCK_SIZE_M": 128,
@@ -195,6 +195,6 @@
195 "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 - "kpack": 1
198 + "kpack": 2
199 }
200 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 16,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 64,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 4,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 128,
23 + "BLOCK_SIZE_K": 128,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 8,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 64,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 4,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 256,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 4,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 64,
50 + "BLOCK_SIZE_K": 256,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 2,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 64,
59 + "BLOCK_SIZE_K": 256,
60 + "GROUP_SIZE_M": 1,
61 + "num_warps": 2,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 64,
68 + "BLOCK_SIZE_K": 256,
69 + "GROUP_SIZE_M": 1,
70 + "num_warps": 4,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 32,
85 + "BLOCK_SIZE_N": 128,
86 + "BLOCK_SIZE_K": 256,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 4,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 16,
94 + "BLOCK_SIZE_N": 64,
95 + "BLOCK_SIZE_K": 256,
96 + "GROUP_SIZE_M": 1,
97 + "num_warps": 4,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 64,
103 + "BLOCK_SIZE_N": 64,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 1,
106 + "num_warps": 4,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 64,
112 + "BLOCK_SIZE_N": 64,
113 + "BLOCK_SIZE_K": 128,
114 + "GROUP_SIZE_M": 1,
115 + "num_warps": 4,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 64,
121 + "BLOCK_SIZE_N": 128,
122 + "BLOCK_SIZE_K": 128,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 4,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 128,
130 + "BLOCK_SIZE_N": 128,
131 + "BLOCK_SIZE_K": 256,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 8,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 256,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 8,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 128,
150 + "GROUP_SIZE_M": 1,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 256,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 64,
159 + "GROUP_SIZE_M": 1,
160 + "num_warps": 8,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=AMD_Instinct_MI300X.json new
+200
@@ -0,0 +1,200 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 2,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 1
12 + },
13 + "2": {
14 + "BLOCK_SIZE_M": 16,
15 + "BLOCK_SIZE_N": 64,
16 + "BLOCK_SIZE_K": 128,
17 + "GROUP_SIZE_M": 1,
18 + "num_warps": 4,
19 + "num_stages": 2,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 2
23 + },
24 + "4": {
25 + "BLOCK_SIZE_M": 16,
26 + "BLOCK_SIZE_N": 32,
27 + "BLOCK_SIZE_K": 128,
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 2,
30 + "num_stages": 2,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 2
34 + },
35 + "8": {
36 + "BLOCK_SIZE_M": 16,
37 + "BLOCK_SIZE_N": 64,
38 + "BLOCK_SIZE_K": 128,
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 4,
41 + "num_stages": 2,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 2
45 + },
46 + "16": {
47 + "BLOCK_SIZE_M": 32,
48 + "BLOCK_SIZE_N": 64,
49 + "BLOCK_SIZE_K": 128,
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 8,
52 + "num_stages": 2,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 1
56 + },
57 + "24": {
58 + "BLOCK_SIZE_M": 16,
59 + "BLOCK_SIZE_N": 64,
60 + "BLOCK_SIZE_K": 128,
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 2,
63 + "num_stages": 2,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 1
67 + },
68 + "32": {
69 + "BLOCK_SIZE_M": 16,
70 + "BLOCK_SIZE_N": 64,
71 + "BLOCK_SIZE_K": 256,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 4,
74 + "num_stages": 2,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 2
78 + },
79 + "48": {
80 + "BLOCK_SIZE_M": 16,
81 + "BLOCK_SIZE_N": 64,
82 + "BLOCK_SIZE_K": 128,
83 + "GROUP_SIZE_M": 1,
84 + "num_warps": 2,
85 + "num_stages": 2,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 2
89 + },
90 + "64": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 4,
95 + "num_warps": 8,
96 + "num_stages": 2,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 2
100 + },
101 + "96": {
102 + "BLOCK_SIZE_M": 32,
103 + "BLOCK_SIZE_N": 32,
104 + "BLOCK_SIZE_K": 128,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 2,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 1
111 + },
112 + "128": {
113 + "BLOCK_SIZE_M": 64,
114 + "BLOCK_SIZE_N": 64,
115 + "BLOCK_SIZE_K": 256,
116 + "GROUP_SIZE_M": 4,
117 + "num_warps": 8,
118 + "num_stages": 2,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 2
122 + },
123 + "256": {
124 + "BLOCK_SIZE_M": 64,
125 + "BLOCK_SIZE_N": 64,
126 + "BLOCK_SIZE_K": 256,
127 + "GROUP_SIZE_M": 1,
128 + "num_warps": 8,
129 + "num_stages": 2,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 16,
132 + "kpack": 2
133 + },
134 + "512": {
135 + "BLOCK_SIZE_M": 64,
136 + "BLOCK_SIZE_N": 128,
137 + "BLOCK_SIZE_K": 128,
138 + "GROUP_SIZE_M": 1,
139 + "num_warps": 8,
140 + "num_stages": 2,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 32,
143 + "kpack": 2
144 + },
145 + "1024": {
146 + "BLOCK_SIZE_M": 128,
147 + "BLOCK_SIZE_N": 128,
148 + "BLOCK_SIZE_K": 64,
149 + "GROUP_SIZE_M": 1,
150 + "num_warps": 8,
151 + "num_stages": 2,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 16,
154 + "kpack": 2
155 + },
156 + "1536": {
157 + "BLOCK_SIZE_M": 128,
158 + "BLOCK_SIZE_N": 128,
159 + "BLOCK_SIZE_K": 64,
160 + "GROUP_SIZE_M": 1,
161 + "num_warps": 8,
162 + "num_stages": 2,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 2
166 + },
167 + "2048": {
168 + "BLOCK_SIZE_M": 128,
169 + "BLOCK_SIZE_N": 128,
170 + "BLOCK_SIZE_K": 64,
171 + "GROUP_SIZE_M": 1,
172 + "num_warps": 8,
173 + "num_stages": 2,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 2
177 + },
178 + "3072": {
179 + "BLOCK_SIZE_M": 128,
180 + "BLOCK_SIZE_N": 128,
181 + "BLOCK_SIZE_K": 64,
182 + "GROUP_SIZE_M": 1,
183 + "num_warps": 8,
184 + "num_stages": 2,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 2
188 + },
189 + "4096": {
190 + "BLOCK_SIZE_M": 128,
191 + "BLOCK_SIZE_N": 128,
192 + "BLOCK_SIZE_K": 64,
193 + "GROUP_SIZE_M": 1,
194 + "num_warps": 8,
195 + "num_stages": 2,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 2
199 + }
200 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 64,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 4,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 256,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 4,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 64,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 2,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 256,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 2,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 64,
50 + "BLOCK_SIZE_K": 256,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 4,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 64,
59 + "BLOCK_SIZE_K": 256,
60 + "GROUP_SIZE_M": 1,
61 + "num_warps": 4,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 64,
68 + "BLOCK_SIZE_K": 256,
69 + "GROUP_SIZE_M": 1,
70 + "num_warps": 2,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 2,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 32,
85 + "BLOCK_SIZE_N": 128,
86 + "BLOCK_SIZE_K": 128,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 4,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 128,
95 + "BLOCK_SIZE_K": 128,
96 + "GROUP_SIZE_M": 4,
97 + "num_warps": 8,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 64,
103 + "BLOCK_SIZE_N": 64,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 1,
106 + "num_warps": 4,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 64,
112 + "BLOCK_SIZE_N": 128,
113 + "BLOCK_SIZE_K": 128,
114 + "GROUP_SIZE_M": 1,
115 + "num_warps": 8,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 64,
121 + "BLOCK_SIZE_N": 128,
122 + "BLOCK_SIZE_K": 128,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 4,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 128,
130 + "BLOCK_SIZE_N": 128,
131 + "BLOCK_SIZE_K": 256,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 8,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 128,
140 + "BLOCK_SIZE_K": 256,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 8,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 128,
150 + "GROUP_SIZE_M": 4,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 128,
157 + "BLOCK_SIZE_N": 128,
158 + "BLOCK_SIZE_K": 128,
159 + "GROUP_SIZE_M": 1,
160 + "num_warps": 4,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json
+35 -35
@@ -1,21 +1,21 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 16,
5 - "BLOCK_SIZE_K": 128,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 - "num_warps": 2,
7 + "num_warps": 4,
8 "num_stages": 2,
9 "waves_per_eu": 0,
10 "matrix_instr_nonkdim": 16,
11 - "kpack": 1
11 + "kpack": 2
12 },
13 "2": {
14 "BLOCK_SIZE_M": 16,
15 - "BLOCK_SIZE_N": 16,
16 - "BLOCK_SIZE_K": 64,
15 + "BLOCK_SIZE_N": 64,
16 + "BLOCK_SIZE_K": 128,
17 "GROUP_SIZE_M": 1,
18 - "num_warps": 2,
18 + "num_warps": 4,
19 "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
@@ -23,10 +23,10 @@
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
26 - "BLOCK_SIZE_N": 32,
27 - "BLOCK_SIZE_K": 256,
26 + "BLOCK_SIZE_N": 64,
27 + "BLOCK_SIZE_K": 128,
28 "GROUP_SIZE_M": 1,
29 - "num_warps": 2,
29 + "num_warps": 4,
30 "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
@@ -34,7 +34,7 @@
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
37 - "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_N": 16,
38 "BLOCK_SIZE_K": 256,
39 "GROUP_SIZE_M": 1,
40 "num_warps": 2,
@@ -52,32 +52,32 @@
52 "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
55 - "kpack": 2
55 + "kpack": 1
56 },
57 "24": {
58 "BLOCK_SIZE_M": 16,
59 - "BLOCK_SIZE_N": 64,
60 - "BLOCK_SIZE_K": 64,
59 + "BLOCK_SIZE_N": 32,
60 + "BLOCK_SIZE_K": 256,
61 "GROUP_SIZE_M": 1,
62 - "num_warps": 4,
62 + "num_warps": 2,
63 "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
66 - "kpack": 1
66 + "kpack": 2
67 },
68 "32": {
69 "BLOCK_SIZE_M": 16,
70 - "BLOCK_SIZE_N": 16,
70 + "BLOCK_SIZE_N": 32,
71 "BLOCK_SIZE_K": 256,
72 "GROUP_SIZE_M": 4,
73 "num_warps": 2,
74 "num_stages": 2,
75 "waves_per_eu": 0,
76 "matrix_instr_nonkdim": 16,
77 - "kpack": 2
77 + "kpack": 1
78 },
79 "48": {
80 - "BLOCK_SIZE_M": 16,
80 + "BLOCK_SIZE_M": 32,
81 "BLOCK_SIZE_N": 32,
82 "BLOCK_SIZE_K": 256,
83 "GROUP_SIZE_M": 1,
@@ -85,7 +85,7 @@
85 "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 - "kpack": 2
88 + "kpack": 1
89 },
90 "64": {
91 "BLOCK_SIZE_M": 32,
@@ -101,40 +101,40 @@
101 "96": {
102 "BLOCK_SIZE_M": 32,
103 "BLOCK_SIZE_N": 32,
104 - "BLOCK_SIZE_K": 128,
104 + "BLOCK_SIZE_K": 256,
105 "GROUP_SIZE_M": 4,
106 - "num_warps": 4,
106 + "num_warps": 2,
107 "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
110 - "kpack": 1
110 + "kpack": 2
111 },
112 "128": {
113 "BLOCK_SIZE_M": 64,
114 "BLOCK_SIZE_N": 64,
115 - "BLOCK_SIZE_K": 128,
116 - "GROUP_SIZE_M": 4,
115 + "BLOCK_SIZE_K": 256,
116 + "GROUP_SIZE_M": 1,
117 "num_warps": 8,
118 "num_stages": 2,
119 "waves_per_eu": 0,
120 "matrix_instr_nonkdim": 16,
121 - "kpack": 1
121 + "kpack": 2
122 },
123 "256": {
124 - "BLOCK_SIZE_M": 128,
125 - "BLOCK_SIZE_N": 128,
126 - "BLOCK_SIZE_K": 64,
127 - "GROUP_SIZE_M": 4,
124 + "BLOCK_SIZE_M": 64,
125 + "BLOCK_SIZE_N": 64,
126 + "BLOCK_SIZE_K": 128,
127 + "GROUP_SIZE_M": 1,
128 "num_warps": 8,
129 "num_stages": 2,
130 "waves_per_eu": 0,
131 "matrix_instr_nonkdim": 16,
132 - "kpack": 1
132 + "kpack": 2
133 },
134 "512": {
135 "BLOCK_SIZE_M": 64,
136 "BLOCK_SIZE_N": 128,
137 - "BLOCK_SIZE_K": 64,
137 + "BLOCK_SIZE_K": 128,
138 "GROUP_SIZE_M": 1,
139 "num_warps": 8,
140 "num_stages": 2,
@@ -151,7 +151,7 @@
151 "num_stages": 2,
152 "waves_per_eu": 0,
153 "matrix_instr_nonkdim": 16,
154 - "kpack": 1
154 + "kpack": 2
155 },
156 "1536": {
157 "BLOCK_SIZE_M": 128,
@@ -173,7 +173,7 @@
173 "num_stages": 2,
174 "waves_per_eu": 0,
175 "matrix_instr_nonkdim": 16,
176 - "kpack": 1
176 + "kpack": 2
177 },
178 "3072": {
179 "BLOCK_SIZE_M": 128,
@@ -195,6 +195,6 @@
195 "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 - "kpack": 1
198 + "kpack": 2
199 }
200 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 128,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 8,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 256,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 2,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 64,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 4,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 128,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 4,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 64,
50 + "BLOCK_SIZE_K": 128,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 4,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 64,
59 + "BLOCK_SIZE_K": 128,
60 + "GROUP_SIZE_M": 1,
61 + "num_warps": 2,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 128,
68 + "BLOCK_SIZE_K": 128,
69 + "GROUP_SIZE_M": 1,
70 + "num_warps": 8,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 2,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 32,
85 + "BLOCK_SIZE_N": 16,
86 + "BLOCK_SIZE_K": 256,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 4,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 64,
95 + "BLOCK_SIZE_K": 128,
96 + "GROUP_SIZE_M": 4,
97 + "num_warps": 4,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 64,
103 + "BLOCK_SIZE_N": 128,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 1,
106 + "num_warps": 8,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 64,
112 + "BLOCK_SIZE_N": 128,
113 + "BLOCK_SIZE_K": 128,
114 + "GROUP_SIZE_M": 1,
115 + "num_warps": 8,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 128,
121 + "BLOCK_SIZE_N": 128,
122 + "BLOCK_SIZE_K": 256,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 8,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 128,
130 + "BLOCK_SIZE_N": 128,
131 + "BLOCK_SIZE_K": 256,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 8,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 128,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 4,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 128,
150 + "GROUP_SIZE_M": 1,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 256,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 64,
159 + "GROUP_SIZE_M": 1,
160 + "num_warps": 8,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X.json new
+200
@@ -0,0 +1,200 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 16,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 2,
8 + "num_stages": 2,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 2
12 + },
13 + "2": {
14 + "BLOCK_SIZE_M": 16,
15 + "BLOCK_SIZE_N": 16,
16 + "BLOCK_SIZE_K": 128,
17 + "GROUP_SIZE_M": 1,
18 + "num_warps": 2,
19 + "num_stages": 2,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 2
23 + },
24 + "4": {
25 + "BLOCK_SIZE_M": 16,
26 + "BLOCK_SIZE_N": 16,
27 + "BLOCK_SIZE_K": 256,
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 4,
30 + "num_stages": 2,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 1
34 + },
35 + "8": {
36 + "BLOCK_SIZE_M": 16,
37 + "BLOCK_SIZE_N": 16,
38 + "BLOCK_SIZE_K": 256,
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 4,
41 + "num_stages": 2,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 2
45 + },
46 + "16": {
47 + "BLOCK_SIZE_M": 16,
48 + "BLOCK_SIZE_N": 16,
49 + "BLOCK_SIZE_K": 256,
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 4,
52 + "num_stages": 2,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 2
56 + },
57 + "24": {
58 + "BLOCK_SIZE_M": 16,
59 + "BLOCK_SIZE_N": 16,
60 + "BLOCK_SIZE_K": 256,
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 4,
63 + "num_stages": 2,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 1
67 + },
68 + "32": {
69 + "BLOCK_SIZE_M": 16,
70 + "BLOCK_SIZE_N": 32,
71 + "BLOCK_SIZE_K": 256,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 2,
74 + "num_stages": 2,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 2
78 + },
79 + "48": {
80 + "BLOCK_SIZE_M": 32,
81 + "BLOCK_SIZE_N": 64,
82 + "BLOCK_SIZE_K": 64,
83 + "GROUP_SIZE_M": 1,
84 + "num_warps": 4,
85 + "num_stages": 2,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 2
89 + },
90 + "64": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 256,
94 + "GROUP_SIZE_M": 4,
95 + "num_warps": 4,
96 + "num_stages": 2,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 1
100 + },
101 + "96": {
102 + "BLOCK_SIZE_M": 32,
103 + "BLOCK_SIZE_N": 64,
104 + "BLOCK_SIZE_K": 64,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 2,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 2
111 + },
112 + "128": {
113 + "BLOCK_SIZE_M": 64,
114 + "BLOCK_SIZE_N": 64,
115 + "BLOCK_SIZE_K": 64,
116 + "GROUP_SIZE_M": 1,
117 + "num_warps": 4,
118 + "num_stages": 2,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 1
122 + },
123 + "256": {
124 + "BLOCK_SIZE_M": 64,
125 + "BLOCK_SIZE_N": 128,
126 + "BLOCK_SIZE_K": 128,
127 + "GROUP_SIZE_M": 1,
128 + "num_warps": 8,
129 + "num_stages": 2,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 32,
132 + "kpack": 2
133 + },
134 + "512": {
135 + "BLOCK_SIZE_M": 128,
136 + "BLOCK_SIZE_N": 128,
137 + "BLOCK_SIZE_K": 64,
138 + "GROUP_SIZE_M": 1,
139 + "num_warps": 8,
140 + "num_stages": 2,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 16,
143 + "kpack": 2
144 + },
145 + "1024": {
146 + "BLOCK_SIZE_M": 128,
147 + "BLOCK_SIZE_N": 128,
148 + "BLOCK_SIZE_K": 64,
149 + "GROUP_SIZE_M": 1,
150 + "num_warps": 8,
151 + "num_stages": 2,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 16,
154 + "kpack": 2
155 + },
156 + "1536": {
157 + "BLOCK_SIZE_M": 128,
158 + "BLOCK_SIZE_N": 128,
159 + "BLOCK_SIZE_K": 64,
160 + "GROUP_SIZE_M": 1,
161 + "num_warps": 8,
162 + "num_stages": 2,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 2
166 + },
167 + "2048": {
168 + "BLOCK_SIZE_M": 128,
169 + "BLOCK_SIZE_N": 128,
170 + "BLOCK_SIZE_K": 64,
171 + "GROUP_SIZE_M": 1,
172 + "num_warps": 8,
173 + "num_stages": 2,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 2
177 + },
178 + "3072": {
179 + "BLOCK_SIZE_M": 128,
180 + "BLOCK_SIZE_N": 128,
181 + "BLOCK_SIZE_K": 64,
182 + "GROUP_SIZE_M": 1,
183 + "num_warps": 8,
184 + "num_stages": 2,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 2
188 + },
189 + "4096": {
190 + "BLOCK_SIZE_M": 128,
191 + "BLOCK_SIZE_N": 128,
192 + "BLOCK_SIZE_K": 64,
193 + "GROUP_SIZE_M": 1,
194 + "num_warps": 8,
195 + "num_stages": 2,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 2
199 + }
200 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 64,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 4,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 128,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 4,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 64,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 4,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 32,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 256,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 2,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 32,
49 + "BLOCK_SIZE_N": 64,
50 + "BLOCK_SIZE_K": 256,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 2,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 32,
58 + "BLOCK_SIZE_N": 64,
59 + "BLOCK_SIZE_K": 256,
60 + "GROUP_SIZE_M": 1,
61 + "num_warps": 2,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 32,
67 + "BLOCK_SIZE_N": 128,
68 + "BLOCK_SIZE_K": 128,
69 + "GROUP_SIZE_M": 1,
70 + "num_warps": 4,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 4,
79 + "num_warps": 2,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 64,
85 + "BLOCK_SIZE_N": 64,
86 + "BLOCK_SIZE_K": 256,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 4,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 64,
95 + "BLOCK_SIZE_K": 256,
96 + "GROUP_SIZE_M": 4,
97 + "num_warps": 4,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 128,
103 + "BLOCK_SIZE_N": 128,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 8,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 64,
112 + "BLOCK_SIZE_N": 128,
113 + "BLOCK_SIZE_K": 128,
114 + "GROUP_SIZE_M": 1,
115 + "num_warps": 8,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 128,
121 + "BLOCK_SIZE_N": 128,
122 + "BLOCK_SIZE_K": 128,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 4,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 128,
130 + "BLOCK_SIZE_N": 128,
131 + "BLOCK_SIZE_K": 256,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 8,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 256,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 8,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 128,
150 + "GROUP_SIZE_M": 1,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 128,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 128,
159 + "GROUP_SIZE_M": 1,
160 + "num_warps": 8,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json
+38 -38
@@ -1,7 +1,7 @@
1 {
2 "1": {
3 "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 16,
4 + "BLOCK_SIZE_N": 32,
5 "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 "num_warps": 2,
@@ -12,54 +12,54 @@
12 },
13 "2": {
14 "BLOCK_SIZE_M": 16,
15 - "BLOCK_SIZE_N": 64,
16 - "BLOCK_SIZE_K": 32,
15 + "BLOCK_SIZE_N": 16,
16 + "BLOCK_SIZE_K": 256,
17 "GROUP_SIZE_M": 1,
18 - "num_warps": 4,
18 + "num_warps": 2,
19 "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
22 - "kpack": 1
22 + "kpack": 2
23 },
24 "4": {
25 "BLOCK_SIZE_M": 16,
26 - "BLOCK_SIZE_N": 32,
27 - "BLOCK_SIZE_K": 128,
26 + "BLOCK_SIZE_N": 16,
27 + "BLOCK_SIZE_K": 256,
28 "GROUP_SIZE_M": 1,
29 - "num_warps": 4,
29 + "num_warps": 2,
30 "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
33 - "kpack": 1
33 + "kpack": 2
34 },
35 "8": {
36 "BLOCK_SIZE_M": 16,
37 - "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_N": 16,
38 "BLOCK_SIZE_K": 256,
39 "GROUP_SIZE_M": 1,
40 - "num_warps": 2,
40 + "num_warps": 1,
41 "num_stages": 2,
42 "waves_per_eu": 0,
43 "matrix_instr_nonkdim": 16,
44 - "kpack": 1
44 + "kpack": 2
45 },
46 "16": {
47 "BLOCK_SIZE_M": 16,
48 - "BLOCK_SIZE_N": 16,
49 - "BLOCK_SIZE_K": 256,
48 + "BLOCK_SIZE_N": 64,
49 + "BLOCK_SIZE_K": 64,
50 "GROUP_SIZE_M": 1,
51 - "num_warps": 4,
51 + "num_warps": 2,
52 "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
55 - "kpack": 2
55 + "kpack": 1
56 },
57 "24": {
58 - "BLOCK_SIZE_M": 32,
59 - "BLOCK_SIZE_N": 32,
60 - "BLOCK_SIZE_K": 128,
58 + "BLOCK_SIZE_M": 16,
59 + "BLOCK_SIZE_N": 16,
60 + "BLOCK_SIZE_K": 256,
61 "GROUP_SIZE_M": 1,
62 - "num_warps": 8,
62 + "num_warps": 1,
63 "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
@@ -68,7 +68,7 @@
68 "32": {
69 "BLOCK_SIZE_M": 16,
70 "BLOCK_SIZE_N": 32,
71 - "BLOCK_SIZE_K": 128,
71 + "BLOCK_SIZE_K": 256,
72 "GROUP_SIZE_M": 4,
73 "num_warps": 2,
74 "num_stages": 2,
@@ -78,32 +78,32 @@
78 },
79 "48": {
80 "BLOCK_SIZE_M": 16,
81 - "BLOCK_SIZE_N": 32,
82 - "BLOCK_SIZE_K": 128,
83 - "GROUP_SIZE_M": 4,
81 + "BLOCK_SIZE_N": 16,
82 + "BLOCK_SIZE_K": 256,
83 + "GROUP_SIZE_M": 1,
84 "num_warps": 2,
85 "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 - "kpack": 1
88 + "kpack": 2
89 },
90 "64": {
91 "BLOCK_SIZE_M": 32,
92 - "BLOCK_SIZE_N": 32,
92 + "BLOCK_SIZE_N": 64,
93 "BLOCK_SIZE_K": 128,
94 "GROUP_SIZE_M": 4,
95 "num_warps": 4,
96 "num_stages": 2,
97 "waves_per_eu": 0,
98 "matrix_instr_nonkdim": 16,
99 - "kpack": 2
99 + "kpack": 1
100 },
101 "96": {
102 "BLOCK_SIZE_M": 32,
103 - "BLOCK_SIZE_N": 32,
104 - "BLOCK_SIZE_K": 128,
103 + "BLOCK_SIZE_N": 64,
104 + "BLOCK_SIZE_K": 256,
105 "GROUP_SIZE_M": 4,
106 - "num_warps": 4,
106 + "num_warps": 8,
107 "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
@@ -112,18 +112,18 @@
112 "128": {
113 "BLOCK_SIZE_M": 64,
114 "BLOCK_SIZE_N": 64,
115 - "BLOCK_SIZE_K": 64,
115 + "BLOCK_SIZE_K": 128,
116 "GROUP_SIZE_M": 4,
117 "num_warps": 8,
118 "num_stages": 2,
119 "waves_per_eu": 0,
120 "matrix_instr_nonkdim": 16,
121 - "kpack": 1
121 + "kpack": 2
122 },
123 "256": {
124 "BLOCK_SIZE_M": 128,
125 "BLOCK_SIZE_N": 128,
126 - "BLOCK_SIZE_K": 64,
126 + "BLOCK_SIZE_K": 128,
127 "GROUP_SIZE_M": 4,
128 "num_warps": 8,
129 "num_stages": 2,
@@ -140,7 +140,7 @@
140 "num_stages": 2,
141 "waves_per_eu": 0,
142 "matrix_instr_nonkdim": 16,
143 - "kpack": 1
143 + "kpack": 2
144 },
145 "1024": {
146 "BLOCK_SIZE_M": 128,
@@ -151,7 +151,7 @@
151 "num_stages": 2,
152 "waves_per_eu": 0,
153 "matrix_instr_nonkdim": 16,
154 - "kpack": 1
154 + "kpack": 2
155 },
156 "1536": {
157 "BLOCK_SIZE_M": 128,
@@ -173,7 +173,7 @@
173 "num_stages": 2,
174 "waves_per_eu": 0,
175 "matrix_instr_nonkdim": 16,
176 - "kpack": 1
176 + "kpack": 2
177 },
178 "3072": {
179 "BLOCK_SIZE_M": 128,
@@ -187,7 +187,7 @@
187 "kpack": 2
188 },
189 "4096": {
190 - "BLOCK_SIZE_M": 128,
190 + "BLOCK_SIZE_M": 256,
191 "BLOCK_SIZE_N": 128,
192 "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
@@ -195,6 +195,6 @@
195 "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 - "kpack": 1
198 + "kpack": 2
199 }
200 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new
+164
@@ -0,0 +1,164 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 2,
8 + "num_stages": 2,
9 + "waves_per_eu": 0
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 16,
13 + "BLOCK_SIZE_N": 128,
14 + "BLOCK_SIZE_K": 256,
15 + "GROUP_SIZE_M": 1,
16 + "num_warps": 4,
17 + "num_stages": 2,
18 + "waves_per_eu": 0
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 32,
23 + "BLOCK_SIZE_K": 128,
24 + "GROUP_SIZE_M": 1,
25 + "num_warps": 2,
26 + "num_stages": 2,
27 + "waves_per_eu": 0
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 64,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 2,
35 + "num_stages": 2,
36 + "waves_per_eu": 0
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 256,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 1,
44 + "num_stages": 2,
45 + "waves_per_eu": 0
46 + },
47 + "24": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 64,
50 + "BLOCK_SIZE_K": 256,
51 + "GROUP_SIZE_M": 1,
52 + "num_warps": 2,
53 + "num_stages": 2,
54 + "waves_per_eu": 0
55 + },
56 + "32": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 64,
59 + "BLOCK_SIZE_K": 256,
60 + "GROUP_SIZE_M": 4,
61 + "num_warps": 4,
62 + "num_stages": 2,
63 + "waves_per_eu": 0
64 + },
65 + "48": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 64,
68 + "BLOCK_SIZE_K": 256,
69 + "GROUP_SIZE_M": 1,
70 + "num_warps": 2,
71 + "num_stages": 2,
72 + "waves_per_eu": 0
73 + },
74 + "64": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 4,
79 + "num_warps": 1,
80 + "num_stages": 2,
81 + "waves_per_eu": 0
82 + },
83 + "96": {
84 + "BLOCK_SIZE_M": 64,
85 + "BLOCK_SIZE_N": 64,
86 + "BLOCK_SIZE_K": 128,
87 + "GROUP_SIZE_M": 1,
88 + "num_warps": 4,
89 + "num_stages": 2,
90 + "waves_per_eu": 0
91 + },
92 + "128": {
93 + "BLOCK_SIZE_M": 64,
94 + "BLOCK_SIZE_N": 64,
95 + "BLOCK_SIZE_K": 128,
96 + "GROUP_SIZE_M": 4,
97 + "num_warps": 2,
98 + "num_stages": 2,
99 + "waves_per_eu": 0
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 64,
103 + "BLOCK_SIZE_N": 64,
104 + "BLOCK_SIZE_K": 128,
105 + "GROUP_SIZE_M": 1,
106 + "num_warps": 4,
107 + "num_stages": 2,
108 + "waves_per_eu": 0
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 128,
112 + "BLOCK_SIZE_N": 128,
113 + "BLOCK_SIZE_K": 256,
114 + "GROUP_SIZE_M": 1,
115 + "num_warps": 8,
116 + "num_stages": 2,
117 + "waves_per_eu": 0
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 128,
121 + "BLOCK_SIZE_N": 256,
122 + "BLOCK_SIZE_K": 128,
123 + "GROUP_SIZE_M": 1,
124 + "num_warps": 8,
125 + "num_stages": 2,
126 + "waves_per_eu": 0
127 + },
128 + "1536": {
129 + "BLOCK_SIZE_M": 128,
130 + "BLOCK_SIZE_N": 256,
131 + "BLOCK_SIZE_K": 128,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 8,
134 + "num_stages": 2,
135 + "waves_per_eu": 0
136 + },
137 + "2048": {
138 + "BLOCK_SIZE_M": 128,
139 + "BLOCK_SIZE_N": 256,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 8,
143 + "num_stages": 2,
144 + "waves_per_eu": 0
145 + },
146 + "3072": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 128,
150 + "GROUP_SIZE_M": 1,
151 + "num_warps": 8,
152 + "num_stages": 2,
153 + "waves_per_eu": 0
154 + },
155 + "4096": {
156 + "BLOCK_SIZE_M": 256,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 64,
159 + "GROUP_SIZE_M": 1,
160 + "num_warps": 8,
161 + "num_stages": 2,
162 + "waves_per_eu": 0
163 + }
164 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X.json new
+200
@@ -0,0 +1,200 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 16,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 1,
8 + "num_stages": 2,
9 + "waves_per_eu": 0,
10 + "matrix_instr_nonkdim": 16,
11 + "kpack": 1
12 + },
13 + "2": {
14 + "BLOCK_SIZE_M": 16,
15 + "BLOCK_SIZE_N": 64,
16 + "BLOCK_SIZE_K": 128,
17 + "GROUP_SIZE_M": 1,
18 + "num_warps": 2,
19 + "num_stages": 2,
20 + "waves_per_eu": 0,
21 + "matrix_instr_nonkdim": 16,
22 + "kpack": 2
23 + },
24 + "4": {
25 + "BLOCK_SIZE_M": 16,
26 + "BLOCK_SIZE_N": 64,
27 + "BLOCK_SIZE_K": 128,
28 + "GROUP_SIZE_M": 1,
29 + "num_warps": 4,
30 + "num_stages": 2,
31 + "waves_per_eu": 0,
32 + "matrix_instr_nonkdim": 16,
33 + "kpack": 2
34 + },
35 + "8": {
36 + "BLOCK_SIZE_M": 16,
37 + "BLOCK_SIZE_N": 16,
38 + "BLOCK_SIZE_K": 256,
39 + "GROUP_SIZE_M": 1,
40 + "num_warps": 1,
41 + "num_stages": 2,
42 + "waves_per_eu": 0,
43 + "matrix_instr_nonkdim": 16,
44 + "kpack": 1
45 + },
46 + "16": {
47 + "BLOCK_SIZE_M": 16,
48 + "BLOCK_SIZE_N": 32,
49 + "BLOCK_SIZE_K": 64,
50 + "GROUP_SIZE_M": 1,
51 + "num_warps": 2,
52 + "num_stages": 2,
53 + "waves_per_eu": 0,
54 + "matrix_instr_nonkdim": 16,
55 + "kpack": 2
56 + },
57 + "24": {
58 + "BLOCK_SIZE_M": 16,
59 + "BLOCK_SIZE_N": 64,
60 + "BLOCK_SIZE_K": 128,
61 + "GROUP_SIZE_M": 1,
62 + "num_warps": 2,
63 + "num_stages": 2,
64 + "waves_per_eu": 0,
65 + "matrix_instr_nonkdim": 16,
66 + "kpack": 2
67 + },
68 + "32": {
69 + "BLOCK_SIZE_M": 16,
70 + "BLOCK_SIZE_N": 64,
71 + "BLOCK_SIZE_K": 128,
72 + "GROUP_SIZE_M": 4,
73 + "num_warps": 2,
74 + "num_stages": 2,
75 + "waves_per_eu": 0,
76 + "matrix_instr_nonkdim": 16,
77 + "kpack": 2
78 + },
79 + "48": {
80 + "BLOCK_SIZE_M": 32,
81 + "BLOCK_SIZE_N": 64,
82 + "BLOCK_SIZE_K": 64,
83 + "GROUP_SIZE_M": 1,
84 + "num_warps": 8,
85 + "num_stages": 2,
86 + "waves_per_eu": 0,
87 + "matrix_instr_nonkdim": 16,
88 + "kpack": 2
89 + },
90 + "64": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 256,
94 + "GROUP_SIZE_M": 4,
95 + "num_warps": 4,
96 + "num_stages": 2,
97 + "waves_per_eu": 0,
98 + "matrix_instr_nonkdim": 16,
99 + "kpack": 2
100 + },
101 + "96": {
102 + "BLOCK_SIZE_M": 32,
103 + "BLOCK_SIZE_N": 32,
104 + "BLOCK_SIZE_K": 256,
105 + "GROUP_SIZE_M": 4,
106 + "num_warps": 4,
107 + "num_stages": 2,
108 + "waves_per_eu": 0,
109 + "matrix_instr_nonkdim": 16,
110 + "kpack": 2
111 + },
112 + "128": {
113 + "BLOCK_SIZE_M": 64,
114 + "BLOCK_SIZE_N": 64,
115 + "BLOCK_SIZE_K": 64,
116 + "GROUP_SIZE_M": 4,
117 + "num_warps": 4,
118 + "num_stages": 2,
119 + "waves_per_eu": 0,
120 + "matrix_instr_nonkdim": 16,
121 + "kpack": 2
122 + },
123 + "256": {
124 + "BLOCK_SIZE_M": 128,
125 + "BLOCK_SIZE_N": 128,
126 + "BLOCK_SIZE_K": 64,
127 + "GROUP_SIZE_M": 1,
128 + "num_warps": 4,
129 + "num_stages": 2,
130 + "waves_per_eu": 0,
131 + "matrix_instr_nonkdim": 16,
132 + "kpack": 2
133 + },
134 + "512": {
135 + "BLOCK_SIZE_M": 128,
136 + "BLOCK_SIZE_N": 128,
137 + "BLOCK_SIZE_K": 64,
138 + "GROUP_SIZE_M": 1,
139 + "num_warps": 8,
140 + "num_stages": 2,
141 + "waves_per_eu": 0,
142 + "matrix_instr_nonkdim": 16,
143 + "kpack": 2
144 + },
145 + "1024": {
146 + "BLOCK_SIZE_M": 128,
147 + "BLOCK_SIZE_N": 128,
148 + "BLOCK_SIZE_K": 64,
149 + "GROUP_SIZE_M": 1,
150 + "num_warps": 8,
151 + "num_stages": 2,
152 + "waves_per_eu": 0,
153 + "matrix_instr_nonkdim": 16,
154 + "kpack": 2
155 + },
156 + "1536": {
157 + "BLOCK_SIZE_M": 128,
158 + "BLOCK_SIZE_N": 128,
159 + "BLOCK_SIZE_K": 64,
160 + "GROUP_SIZE_M": 1,
161 + "num_warps": 8,
162 + "num_stages": 2,
163 + "waves_per_eu": 0,
164 + "matrix_instr_nonkdim": 16,
165 + "kpack": 2
166 + },
167 + "2048": {
168 + "BLOCK_SIZE_M": 128,
169 + "BLOCK_SIZE_N": 128,
170 + "BLOCK_SIZE_K": 64,
171 + "GROUP_SIZE_M": 1,
172 + "num_warps": 8,
173 + "num_stages": 2,
174 + "waves_per_eu": 0,
175 + "matrix_instr_nonkdim": 16,
176 + "kpack": 2
177 + },
178 + "3072": {
179 + "BLOCK_SIZE_M": 128,
180 + "BLOCK_SIZE_N": 128,
181 + "BLOCK_SIZE_K": 64,
182 + "GROUP_SIZE_M": 1,
183 + "num_warps": 8,
184 + "num_stages": 2,
185 + "waves_per_eu": 0,
186 + "matrix_instr_nonkdim": 16,
187 + "kpack": 2
188 + },
189 + "4096": {
190 + "BLOCK_SIZE_M": 128,
191 + "BLOCK_SIZE_N": 128,
192 + "BLOCK_SIZE_K": 64,
193 + "GROUP_SIZE_M": 1,
194 + "num_warps": 8,
195 + "num_stages": 2,
196 + "waves_per_eu": 0,
197 + "matrix_instr_nonkdim": 16,
198 + "kpack": 2
199 + }
200 +}