@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd8de11f0a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel][ROCm][AMD] Add fused_moe Triton configs for MI300X (#4951)

This PR adds Triton kernel configs for the MoE kernel for MI300X

Divakar Verma committed May 28, 2024 at 11:03 UTC dd8de11f0a15f9ef48cd1dcac02dd8e2a8bcb494
4 files changed +494
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json new
+128
@@ -0,0 +1,128 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_stages": 0
8 + },
9 + "2": {
10 + "BLOCK_SIZE_M": 16,
11 + "BLOCK_SIZE_N": 64,
12 + "BLOCK_SIZE_K": 128,
13 + "GROUP_SIZE_M": 1,
14 + "num_stages": 0
15 + },
16 + "4": {
17 + "BLOCK_SIZE_M": 16,
18 + "BLOCK_SIZE_N": 64,
19 + "BLOCK_SIZE_K": 256,
20 + "GROUP_SIZE_M": 64,
21 + "num_stages": 1
22 + },
23 + "8": {
24 + "BLOCK_SIZE_M": 16,
25 + "BLOCK_SIZE_N": 64,
26 + "BLOCK_SIZE_K": 256,
27 + "GROUP_SIZE_M": 32,
28 + "num_stages": 1
29 + },
30 + "16": {
31 + "BLOCK_SIZE_M": 16,
32 + "BLOCK_SIZE_N": 64,
33 + "BLOCK_SIZE_K": 256,
34 + "GROUP_SIZE_M": 8,
35 + "num_stages": 1
36 + },
37 + "24": {
38 + "BLOCK_SIZE_M": 16,
39 + "BLOCK_SIZE_N": 64,
40 + "BLOCK_SIZE_K": 256,
41 + "GROUP_SIZE_M": 64,
42 + "num_stages": 1
43 + },
44 + "32": {
45 + "BLOCK_SIZE_M": 16,
46 + "BLOCK_SIZE_N": 128,
47 + "BLOCK_SIZE_K": 256,
48 + "GROUP_SIZE_M": 8,
49 + "num_stages": 1
50 + },
51 + "48": {
52 + "BLOCK_SIZE_M": 16,
53 + "BLOCK_SIZE_N": 64,
54 + "BLOCK_SIZE_K": 128,
55 + "GROUP_SIZE_M": 8,
56 + "num_stages": 0
57 + },
58 + "64": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 8,
63 + "num_stages": 0
64 + },
65 + "96": {
66 + "BLOCK_SIZE_M": 32,
67 + "BLOCK_SIZE_N": 128,
68 + "BLOCK_SIZE_K": 128,
69 + "GROUP_SIZE_M": 16,
70 + "num_stages": 0
71 + },
72 + "128": {
73 + "BLOCK_SIZE_M": 64,
74 + "BLOCK_SIZE_N": 64,
75 + "BLOCK_SIZE_K": 128,
76 + "GROUP_SIZE_M": 8,
77 + "num_stages": 0
78 + },
79 + "256": {
80 + "BLOCK_SIZE_M": 128,
81 + "BLOCK_SIZE_N": 128,
82 + "BLOCK_SIZE_K": 64,
83 + "GROUP_SIZE_M": 8,
84 + "num_stages": 0
85 + },
86 + "512": {
87 + "BLOCK_SIZE_M": 256,
88 + "BLOCK_SIZE_N": 128,
89 + "BLOCK_SIZE_K": 64,
90 + "GROUP_SIZE_M": 8,
91 + "num_stages": 0
92 + },
93 + "1024": {
94 + "BLOCK_SIZE_M": 128,
95 + "BLOCK_SIZE_N": 128,
96 + "BLOCK_SIZE_K": 64,
97 + "GROUP_SIZE_M": 1,
98 + "num_stages": 0
99 + },
100 + "1536": {
101 + "BLOCK_SIZE_M": 128,
102 + "BLOCK_SIZE_N": 128,
103 + "BLOCK_SIZE_K": 64,
104 + "GROUP_SIZE_M": 1,
105 + "num_stages": 0
106 + },
107 + "2048": {
108 + "BLOCK_SIZE_M": 128,
109 + "BLOCK_SIZE_N": 256,
110 + "BLOCK_SIZE_K": 64,
111 + "GROUP_SIZE_M": 1,
112 + "num_stages": 0
113 + },
114 + "3072": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 1,
119 + "num_stages": 0
120 + },
121 + "4096": {
122 + "BLOCK_SIZE_M": 128,
123 + "BLOCK_SIZE_N": 128,
124 + "BLOCK_SIZE_K": 64,
125 + "GROUP_SIZE_M": 1,
126 + "num_stages": 0
127 + }
128 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json new
+110
@@ -0,0 +1,110 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64
7 + },
8 + "2": {
9 + "BLOCK_SIZE_M": 16,
10 + "BLOCK_SIZE_N": 128,
11 + "BLOCK_SIZE_K": 32,
12 + "GROUP_SIZE_M": 32
13 + },
14 + "4": {
15 + "BLOCK_SIZE_M": 16,
16 + "BLOCK_SIZE_N": 32,
17 + "BLOCK_SIZE_K": 64,
18 + "GROUP_SIZE_M": 8
19 + },
20 + "8": {
21 + "BLOCK_SIZE_M": 16,
22 + "BLOCK_SIZE_N": 64,
23 + "BLOCK_SIZE_K": 256,
24 + "GROUP_SIZE_M": 1
25 + },
26 + "16": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 1
31 + },
32 + "24": {
33 + "BLOCK_SIZE_M": 32,
34 + "BLOCK_SIZE_N": 128,
35 + "BLOCK_SIZE_K": 128,
36 + "GROUP_SIZE_M": 1
37 + },
38 + "32": {
39 + "BLOCK_SIZE_M": 64,
40 + "BLOCK_SIZE_N": 64,
41 + "BLOCK_SIZE_K": 64,
42 + "GROUP_SIZE_M": 8
43 + },
44 + "48": {
45 + "BLOCK_SIZE_M": 128,
46 + "BLOCK_SIZE_N": 64,
47 + "BLOCK_SIZE_K": 64,
48 + "GROUP_SIZE_M": 8
49 + },
50 + "64": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1
55 + },
56 + "96": {
57 + "BLOCK_SIZE_M": 32,
58 + "BLOCK_SIZE_N": 64,
59 + "BLOCK_SIZE_K": 128,
60 + "GROUP_SIZE_M": 8
61 + },
62 + "128": {
63 + "BLOCK_SIZE_M": 64,
64 + "BLOCK_SIZE_N": 64,
65 + "BLOCK_SIZE_K": 128,
66 + "GROUP_SIZE_M": 32
67 + },
68 + "256": {
69 + "BLOCK_SIZE_M": 32,
70 + "BLOCK_SIZE_N": 128,
71 + "BLOCK_SIZE_K": 64,
72 + "GROUP_SIZE_M": 1
73 + },
74 + "512": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1
79 + },
80 + "1024": {
81 + "BLOCK_SIZE_M": 64,
82 + "BLOCK_SIZE_N": 128,
83 + "BLOCK_SIZE_K": 64,
84 + "GROUP_SIZE_M": 1
85 + },
86 + "1536": {
87 + "BLOCK_SIZE_M": 64,
88 + "BLOCK_SIZE_N": 64,
89 + "BLOCK_SIZE_K": 128,
90 + "GROUP_SIZE_M": 1
91 + },
92 + "2048": {
93 + "BLOCK_SIZE_M": 128,
94 + "BLOCK_SIZE_N": 128,
95 + "BLOCK_SIZE_K": 64,
96 + "GROUP_SIZE_M": 1
97 + },
98 + "3072": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 1
103 + },
104 + "4096": {
105 + "BLOCK_SIZE_M": 128,
106 + "BLOCK_SIZE_N": 128,
107 + "BLOCK_SIZE_K": 64,
108 + "GROUP_SIZE_M": 1
109 + }
110 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json new
+128
@@ -0,0 +1,128 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 8,
7 + "num_stages": 0
8 + },
9 + "2": {
10 + "BLOCK_SIZE_M": 16,
11 + "BLOCK_SIZE_N": 128,
12 + "BLOCK_SIZE_K": 64,
13 + "GROUP_SIZE_M": 1,
14 + "num_stages": 0
15 + },
16 + "4": {
17 + "BLOCK_SIZE_M": 16,
18 + "BLOCK_SIZE_N": 64,
19 + "BLOCK_SIZE_K": 32,
20 + "GROUP_SIZE_M": 32,
21 + "num_stages": 1
22 + },
23 + "8": {
24 + "BLOCK_SIZE_M": 16,
25 + "BLOCK_SIZE_N": 32,
26 + "BLOCK_SIZE_K": 256,
27 + "GROUP_SIZE_M": 8,
28 + "num_stages": 1
29 + },
30 + "16": {
31 + "BLOCK_SIZE_M": 32,
32 + "BLOCK_SIZE_N": 128,
33 + "BLOCK_SIZE_K": 128,
34 + "GROUP_SIZE_M": 16,
35 + "num_stages": 1
36 + },
37 + "24": {
38 + "BLOCK_SIZE_M": 16,
39 + "BLOCK_SIZE_N": 64,
40 + "BLOCK_SIZE_K": 256,
41 + "GROUP_SIZE_M": 8,
42 + "num_stages": 1
43 + },
44 + "32": {
45 + "BLOCK_SIZE_M": 16,
46 + "BLOCK_SIZE_N": 256,
47 + "BLOCK_SIZE_K": 64,
48 + "GROUP_SIZE_M": 16,
49 + "num_stages": 0
50 + },
51 + "48": {
52 + "BLOCK_SIZE_M": 16,
53 + "BLOCK_SIZE_N": 128,
54 + "BLOCK_SIZE_K": 256,
55 + "GROUP_SIZE_M": 16,
56 + "num_stages": 1
57 + },
58 + "64": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 32,
63 + "num_stages": 0
64 + },
65 + "96": {
66 + "BLOCK_SIZE_M": 32,
67 + "BLOCK_SIZE_N": 32,
68 + "BLOCK_SIZE_K": 64,
69 + "GROUP_SIZE_M": 16,
70 + "num_stages": 0
71 + },
72 + "128": {
73 + "BLOCK_SIZE_M": 64,
74 + "BLOCK_SIZE_N": 256,
75 + "BLOCK_SIZE_K": 64,
76 + "GROUP_SIZE_M": 8,
77 + "num_stages": 0
78 + },
79 + "256": {
80 + "BLOCK_SIZE_M": 128,
81 + "BLOCK_SIZE_N": 128,
82 + "BLOCK_SIZE_K": 64,
83 + "GROUP_SIZE_M": 8,
84 + "num_stages": 0
85 + },
86 + "512": {
87 + "BLOCK_SIZE_M": 64,
88 + "BLOCK_SIZE_N": 64,
89 + "BLOCK_SIZE_K": 128,
90 + "GROUP_SIZE_M": 1,
91 + "num_stages": 0
92 + },
93 + "1024": {
94 + "BLOCK_SIZE_M": 64,
95 + "BLOCK_SIZE_N": 128,
96 + "BLOCK_SIZE_K": 64,
97 + "GROUP_SIZE_M": 1,
98 + "num_stages": 0
99 + },
100 + "1536": {
101 + "BLOCK_SIZE_M": 128,
102 + "BLOCK_SIZE_N": 128,
103 + "BLOCK_SIZE_K": 64,
104 + "GROUP_SIZE_M": 1,
105 + "num_stages": 0
106 + },
107 + "2048": {
108 + "BLOCK_SIZE_M": 128,
109 + "BLOCK_SIZE_N": 128,
110 + "BLOCK_SIZE_K": 64,
111 + "GROUP_SIZE_M": 1,
112 + "num_stages": 0
113 + },
114 + "3072": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 1,
119 + "num_stages": 0
120 + },
121 + "4096": {
122 + "BLOCK_SIZE_M": 128,
123 + "BLOCK_SIZE_N": 128,
124 + "BLOCK_SIZE_K": 64,
125 + "GROUP_SIZE_M": 1,
126 + "num_stages": 0
127 + }
128 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json new
+128
@@ -0,0 +1,128 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_stages": 0
8 + },
9 + "2": {
10 + "BLOCK_SIZE_M": 16,
11 + "BLOCK_SIZE_N": 128,
12 + "BLOCK_SIZE_K": 256,
13 + "GROUP_SIZE_M": 1,
14 + "num_stages": 1
15 + },
16 + "4": {
17 + "BLOCK_SIZE_M": 16,
18 + "BLOCK_SIZE_N": 64,
19 + "BLOCK_SIZE_K": 256,
20 + "GROUP_SIZE_M": 32,
21 + "num_stages": 1
22 + },
23 + "8": {
24 + "BLOCK_SIZE_M": 16,
25 + "BLOCK_SIZE_N": 64,
26 + "BLOCK_SIZE_K": 256,
27 + "GROUP_SIZE_M": 8,
28 + "num_stages": 1
29 + },
30 + "16": {
31 + "BLOCK_SIZE_M": 16,
32 + "BLOCK_SIZE_N": 64,
33 + "BLOCK_SIZE_K": 256,
34 + "GROUP_SIZE_M": 8,
35 + "num_stages": 1
36 + },
37 + "24": {
38 + "BLOCK_SIZE_M": 16,
39 + "BLOCK_SIZE_N": 64,
40 + "BLOCK_SIZE_K": 256,
41 + "GROUP_SIZE_M": 8,
42 + "num_stages": 1
43 + },
44 + "32": {
45 + "BLOCK_SIZE_M": 16,
46 + "BLOCK_SIZE_N": 64,
47 + "BLOCK_SIZE_K": 256,
48 + "GROUP_SIZE_M": 16,
49 + "num_stages": 0
50 + },
51 + "48": {
52 + "BLOCK_SIZE_M": 16,
53 + "BLOCK_SIZE_N": 64,
54 + "BLOCK_SIZE_K": 256,
55 + "GROUP_SIZE_M": 16,
56 + "num_stages": 0
57 + },
58 + "64": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 256,
62 + "GROUP_SIZE_M": 8,
63 + "num_stages": 1
64 + },
65 + "96": {
66 + "BLOCK_SIZE_M": 32,
67 + "BLOCK_SIZE_N": 128,
68 + "BLOCK_SIZE_K": 128,
69 + "GROUP_SIZE_M": 8,
70 + "num_stages": 0
71 + },
72 + "128": {
73 + "BLOCK_SIZE_M": 64,
74 + "BLOCK_SIZE_N": 64,
75 + "BLOCK_SIZE_K": 128,
76 + "GROUP_SIZE_M": 8,
77 + "num_stages": 0
78 + },
79 + "256": {
80 + "BLOCK_SIZE_M": 128,
81 + "BLOCK_SIZE_N": 64,
82 + "BLOCK_SIZE_K": 64,
83 + "GROUP_SIZE_M": 8,
84 + "num_stages": 0
85 + },
86 + "512": {
87 + "BLOCK_SIZE_M": 256,
88 + "BLOCK_SIZE_N": 128,
89 + "BLOCK_SIZE_K": 64,
90 + "GROUP_SIZE_M": 8,
91 + "num_stages": 0
92 + },
93 + "1024": {
94 + "BLOCK_SIZE_M": 128,
95 + "BLOCK_SIZE_N": 128,
96 + "BLOCK_SIZE_K": 64,
97 + "GROUP_SIZE_M": 1,
98 + "num_stages": 0
99 + },
100 + "1536": {
101 + "BLOCK_SIZE_M": 128,
102 + "BLOCK_SIZE_N": 128,
103 + "BLOCK_SIZE_K": 64,
104 + "GROUP_SIZE_M": 1,
105 + "num_stages": 0
106 + },
107 + "2048": {
108 + "BLOCK_SIZE_M": 128,
109 + "BLOCK_SIZE_N": 128,
110 + "BLOCK_SIZE_K": 64,
111 + "GROUP_SIZE_M": 1,
112 + "num_stages": 0
113 + },
114 + "3072": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 1,
119 + "num_stages": 0
120 + },
121 + "4096": {
122 + "BLOCK_SIZE_M": 256,
123 + "BLOCK_SIZE_N": 256,
124 + "BLOCK_SIZE_K": 32,
125 + "GROUP_SIZE_M": 1,
126 + "num_stages": 0
127 + }
128 +}