@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f710090d8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Kernel] adding fused moe kernel config for L40S TP4 (#9245)

Burkhard Ringlein committed Oct 11, 2024 at 08:54 UTC f710090d8e40451879690b6a27b7d3b1a41b53ec
1 file changed +173
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_L40S.json new
+173
@@ -0,0 +1,173 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_ctas": 1,
9 + "num_stages": 2
10 + },
11 + "2": {
12 + "BLOCK_SIZE_M": 32,
13 + "BLOCK_SIZE_N": 64,
14 + "BLOCK_SIZE_K": 32,
15 + "GROUP_SIZE_M": 2,
16 + "num_warps": 4,
17 + "num_ctas": 1,
18 + "num_stages": 7
19 + },
20 + "4": {
21 + "BLOCK_SIZE_M": 32,
22 + "BLOCK_SIZE_N": 128,
23 + "BLOCK_SIZE_K": 32,
24 + "GROUP_SIZE_M": 128,
25 + "num_warps": 2,
26 + "num_ctas": 1,
27 + "num_stages": 4
28 + },
29 + "8": {
30 + "BLOCK_SIZE_M": 16,
31 + "BLOCK_SIZE_N": 32,
32 + "BLOCK_SIZE_K": 256,
33 + "GROUP_SIZE_M": 1,
34 + "num_warps": 2,
35 + "num_ctas": 1,
36 + "num_stages": 1
37 + },
38 + "16": {
39 + "BLOCK_SIZE_M": 16,
40 + "BLOCK_SIZE_N": 32,
41 + "BLOCK_SIZE_K": 256,
42 + "GROUP_SIZE_M": 1,
43 + "num_warps": 2,
44 + "num_ctas": 1,
45 + "num_stages": 1
46 + },
47 + "32": {
48 + "BLOCK_SIZE_M": 16,
49 + "BLOCK_SIZE_N": 256,
50 + "BLOCK_SIZE_K": 128,
51 + "GROUP_SIZE_M": 2,
52 + "num_warps": 4,
53 + "num_ctas": 1,
54 + "num_stages": 2
55 + },
56 + "48": {
57 + "BLOCK_SIZE_M": 16,
58 + "BLOCK_SIZE_N": 256,
59 + "BLOCK_SIZE_K": 128,
60 + "GROUP_SIZE_M": 2,
61 + "num_warps": 4,
62 + "num_ctas": 1,
63 + "num_stages": 2
64 + },
65 + "64": {
66 + "BLOCK_SIZE_M": 16,
67 + "BLOCK_SIZE_N": 256,
68 + "BLOCK_SIZE_K": 128,
69 + "GROUP_SIZE_M": 2,
70 + "num_warps": 4,
71 + "num_ctas": 1,
72 + "num_stages": 2
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 256,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 8,
79 + "num_warps": 8,
80 + "num_ctas": 1,
81 + "num_stages": 2
82 + },
83 + "128": {
84 + "BLOCK_SIZE_M": 32,
85 + "BLOCK_SIZE_N": 256,
86 + "BLOCK_SIZE_K": 128,
87 + "GROUP_SIZE_M": 8,
88 + "num_warps": 8,
89 + "num_ctas": 1,
90 + "num_stages": 2
91 + },
92 + "192": {
93 + "BLOCK_SIZE_M": 32,
94 + "BLOCK_SIZE_N": 256,
95 + "BLOCK_SIZE_K": 128,
96 + "GROUP_SIZE_M": 8,
97 + "num_warps": 8,
98 + "num_ctas": 1,
99 + "num_stages": 2
100 + },
101 + "256": {
102 + "BLOCK_SIZE_M": 64,
103 + "BLOCK_SIZE_N": 256,
104 + "BLOCK_SIZE_K": 128,
105 + "GROUP_SIZE_M": 1,
106 + "num_warps": 16,
107 + "num_ctas": 1,
108 + "num_stages": 2
109 + },
110 + "512": {
111 + "BLOCK_SIZE_M": 32,
112 + "BLOCK_SIZE_N": 32,
113 + "BLOCK_SIZE_K": 64,
114 + "GROUP_SIZE_M": 128,
115 + "num_warps": 2,
116 + "num_ctas": 1,
117 + "num_stages": 8
118 + },
119 + "1024": {
120 + "BLOCK_SIZE_M": 64,
121 + "BLOCK_SIZE_N": 128,
122 + "BLOCK_SIZE_K": 128,
123 + "GROUP_SIZE_M": 4,
124 + "num_warps": 8,
125 + "num_ctas": 1,
126 + "num_stages": 3
127 + },
128 + "2048": {
129 + "BLOCK_SIZE_M": 64,
130 + "BLOCK_SIZE_N": 256,
131 + "BLOCK_SIZE_K": 128,
132 + "GROUP_SIZE_M": 1,
133 + "num_warps": 16,
134 + "num_ctas": 1,
135 + "num_stages": 2
136 + },
137 + "3072": {
138 + "BLOCK_SIZE_M": 64,
139 + "BLOCK_SIZE_N": 256,
140 + "BLOCK_SIZE_K": 128,
141 + "GROUP_SIZE_M": 1,
142 + "num_warps": 16,
143 + "num_ctas": 1,
144 + "num_stages": 2
145 + },
146 + "4096": {
147 + "BLOCK_SIZE_M": 128,
148 + "BLOCK_SIZE_N": 256,
149 + "BLOCK_SIZE_K": 32,
150 + "GROUP_SIZE_M": 64,
151 + "num_warps": 8,
152 + "num_ctas": 1,
153 + "num_stages": 2
154 + },
155 + "6144": {
156 + "BLOCK_SIZE_M": 128,
157 + "BLOCK_SIZE_N": 256,
158 + "BLOCK_SIZE_K": 32,
159 + "GROUP_SIZE_M": 64,
160 + "num_warps": 8,
161 + "num_ctas": 1,
162 + "num_stages": 2
163 + },
164 + "8192": {
165 + "BLOCK_SIZE_M": 64,
166 + "BLOCK_SIZE_N": 256,
167 + "BLOCK_SIZE_K": 128,
168 + "GROUP_SIZE_M": 1,
169 + "num_warps": 16,
170 + "num_ctas": 1,
171 + "num_stages": 2
172 + }
173 +}
\ No newline at end of file