@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 71c1e07107
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[Kernel] Add more tuned configs (#14877)

Signed-off-by: simon-mo <simon.mo@hey.com>

Simon Mo committed Mar 15, 2025 at 20:25 UTC 71c1e0710783e1b0427610ba9e32bed7724fa36f
105 files changed +13627
vllm/model_executor/layers/fused_moe/configs/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 256,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 32,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 32,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 32,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 16,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 32,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 32,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 32,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 8,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 256,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 16,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 16,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 16,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 16,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 32,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 16,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 16,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 16,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 8,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 256,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 8,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 8,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 8,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=1280,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 2
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 8,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 8,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 8,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 256,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 256,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 8,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 8,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=2560,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 2
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 256,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 8,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 8,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 32,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=320,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 32,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 8,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 8,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 8,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 2
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 8,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 256,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 8,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 256,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 8,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 256,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 256,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 8,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 8,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 8,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 8,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 8,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=64,N=640,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 32,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 8,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 8,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 8,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 8,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 8,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 8,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 8,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 8,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 32,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 2
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 8,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 256,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 256,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 256,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 256,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 256,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 8,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 8,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 8,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 8,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 8,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 32,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 8,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 8,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 8,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 8,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 8,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H200.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 8,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 8,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 8,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/fused_moe/configs/README
+3
@@ -8,3 +8,6 @@ the JSON file contains a mapping from M (batch size) to the chosen configuration
8 The example configurations provided are for the Mixtral model for TP2 on H100
9 and TP4 on A100. Mixtral has intermediate size N = 14336, i.e. for TP2 we have
10 N = 7168 and for TP4 we have N = 3584.
11 +
12 +Please feel free to tune the configurations using scripts in `benchmarks/kernels/benchmark_moe.py`
13 +Some of the configurations files are copied from the SGLang repository. Thank you!
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 8,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 8,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 8,
104 + "num_stages": 5
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 32,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 8,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 8,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 8,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 128,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 8,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 8,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 5
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 2
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 8,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 128,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 8,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 8,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 8,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 256,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 8,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 128,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 8,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 8,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 128,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 128,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 8,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 128,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 2
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 8,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 8,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 256,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 8,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 8,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 128,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 8,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 32,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 16,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 5
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_N": 32,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 5
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 5
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 32,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 32,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 5
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 32,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 32,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 5
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 5
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 32,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 8,
24 + "num_stages": 5
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 8,
16 + "num_stages": 2
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 32,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 8,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 8,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 8,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 256,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 128,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 5
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 8,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 8,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 128,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 2
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 8,
16 + "num_stages": 2
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 2
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 2
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 8,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 5
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 8,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 32,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 32,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 8,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 128,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 2
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 8,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 8,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 8,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 128,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 16,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 16,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 16,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 5
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 32,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 4
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 16,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 32,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 32,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 5
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 32,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 32,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 5
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 5
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 32,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 32,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 32,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 5
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 8,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 2
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 8,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 8,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 32,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 256,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 8,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 2
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 8,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 8,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 32,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 8,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 64,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 32,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 32,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 8,
8 + "num_stages": 2
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 8,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 8,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 32,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 8,
48 + "num_stages": 2
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 64,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 8,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 256,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 8,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 8,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 256,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 8,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 8,
88 + "num_stages": 5
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 5
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 8,
8 + "num_stages": 2
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 32,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 64,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 8,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 32,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 8,
32 + "num_stages": 2
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 32,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 8,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 64,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 128,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 2
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 128,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 2
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 2
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 2
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 2
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 2
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 128,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 2
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 256,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 8,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 32,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 128,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 64,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 64,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 32,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 1,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 32,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 32,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 5
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128,128].json new
+26
@@ -0,0 +1,26 @@
1 +{
2 + "2048": {
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 3
9 + },
10 + "3072": {
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4096": {
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + }
26 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 3
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 32,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 5
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 4,
48 + "num_stages": 3
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 2
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 1,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 + "num_warps": 4,
16 + "num_stages": 3
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 64,
31 + "num_warps": 4,
32 + "num_stages": 3
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 5
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 + "num_warps": 4,
64 + "num_stages": 5
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 3
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 3
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 3
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 64,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 4,
104 + "num_stages": 3
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 64,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 4,
120 + "num_stages": 3
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 3
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 1,
135 + "num_warps": 4,
136 + "num_stages": 3
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 64,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 64,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128,128].json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 256,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 5
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 8,
16 + "num_stages": 5
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 5
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 5
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 32,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 + "num_stages": 3
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 + "num_warps": 4,
48 + "num_stages": 5
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 + "num_stages": 3
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 2
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 16,
68 + "BLOCK_SIZE_N": 256,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 64,
71 + "num_warps": 8,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 128,
84 + "BLOCK_SIZE_N": 32,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 2
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 64,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 3
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 128,
102 + "GROUP_SIZE_M": 16,
103 + "num_warps": 4,
104 + "num_stages": 2
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 128,
109 + "BLOCK_SIZE_K": 128,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 4,
112 + "num_stages": 3
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 64,
117 + "BLOCK_SIZE_K": 128,
118 + "GROUP_SIZE_M": 1,
119 + "num_warps": 4,
120 + "num_stages": 2
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 64,
125 + "BLOCK_SIZE_K": 128,
126 + "GROUP_SIZE_M": 1,
127 + "num_warps": 4,
128 + "num_stages": 2
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 64,
132 + "BLOCK_SIZE_N": 64,
133 + "BLOCK_SIZE_K": 128,
134 + "GROUP_SIZE_M": 32,
135 + "num_warps": 4,
136 + "num_stages": 2
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 64,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 128,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 4,
144 + "num_stages": 3
145 + }
146 +}
\ No newline at end of file