@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / abe855d637
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Retune Mixtral 8x22b configs for FP8 on H100 (#5294)

Philipp Moritz committed Jun 6, 2024 at 09:29 UTC abe855d63774c44e69048dfd188f0333db581d4b
2 files changed +91 -91
vllm/model_executor/layers/fused_moe/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
+55 -55
@@ -1,113 +1,113 @@
1 {
2 "1": {
3 - "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 32,
5 - "BLOCK_SIZE_K": 64,
6 - "GROUP_SIZE_M": 1,
7 - "num_warps": 8,
8 - "num_stages": 3
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 4,
8 + "num_stages": 5
9 },
10 "2": {
11 - "BLOCK_SIZE_M": 16,
12 - "BLOCK_SIZE_N": 64,
13 - "BLOCK_SIZE_K": 64,
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 128,
14 "GROUP_SIZE_M": 16,
15 "num_warps": 4,
16 "num_stages": 5
17 },
18 "4": {
19 "BLOCK_SIZE_M": 64,
20 - "BLOCK_SIZE_N": 32,
21 - "BLOCK_SIZE_K": 256,
22 - "GROUP_SIZE_M": 64,
23 - "num_warps": 8,
24 - "num_stages": 5
20 + "BLOCK_SIZE_N": 256,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 },
26 "8": {
27 - "BLOCK_SIZE_M": 16,
28 - "BLOCK_SIZE_N": 64,
29 - "BLOCK_SIZE_K": 256,
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 256,
29 + "BLOCK_SIZE_K": 128,
30 "GROUP_SIZE_M": 1,
31 "num_warps": 4,
32 - "num_stages": 3
32 + "num_stages": 5
33 },
34 "16": {
35 - "BLOCK_SIZE_M": 128,
35 + "BLOCK_SIZE_M": 64,
36 "BLOCK_SIZE_N": 64,
37 - "BLOCK_SIZE_K": 64,
38 - "GROUP_SIZE_M": 1,
39 - "num_warps": 8,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 4,
40 "num_stages": 3
41 },
42 "24": {
43 "BLOCK_SIZE_M": 64,
44 - "BLOCK_SIZE_N": 256,
44 + "BLOCK_SIZE_N": 64,
45 "BLOCK_SIZE_K": 128,
46 - "GROUP_SIZE_M": 64,
47 - "num_warps": 8,
48 - "num_stages": 2
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 3
49 },
50 "32": {
51 "BLOCK_SIZE_M": 64,
52 - "BLOCK_SIZE_N": 256,
53 - "BLOCK_SIZE_K": 64,
54 - "GROUP_SIZE_M": 1,
55 - "num_warps": 8,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 16,
55 + "num_warps": 4,
56 "num_stages": 3
57 },
58 "48": {
59 "BLOCK_SIZE_M": 64,
60 - "BLOCK_SIZE_N": 256,
60 + "BLOCK_SIZE_N": 64,
61 "BLOCK_SIZE_K": 128,
62 - "GROUP_SIZE_M": 64,
63 - "num_warps": 8,
64 - "num_stages": 4
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 3
65 },
66 "64": {
67 - "BLOCK_SIZE_M": 128,
68 - "BLOCK_SIZE_N": 256,
67 + "BLOCK_SIZE_M": 64,
68 + "BLOCK_SIZE_N": 64,
69 "BLOCK_SIZE_K": 128,
70 - "GROUP_SIZE_M": 64,
71 - "num_warps": 8,
72 - "num_stages": 2
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 3
73 },
74 "96": {
75 "BLOCK_SIZE_M": 64,
76 - "BLOCK_SIZE_N": 256,
76 + "BLOCK_SIZE_N": 64,
77 "BLOCK_SIZE_K": 128,
78 "GROUP_SIZE_M": 16,
79 "num_warps": 4,
80 - "num_stages": 4
80 + "num_stages": 3
81 },
82 "128": {
83 "BLOCK_SIZE_M": 64,
84 - "BLOCK_SIZE_N": 256,
85 - "BLOCK_SIZE_K": 64,
86 - "GROUP_SIZE_M": 32,
87 - "num_warps": 8,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 "num_stages": 3
89 },
90 "256": {
91 - "BLOCK_SIZE_M": 128,
92 - "BLOCK_SIZE_N": 64,
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 "BLOCK_SIZE_K": 128,
94 "GROUP_SIZE_M": 64,
95 - "num_warps": 8,
96 - "num_stages": 4
95 + "num_warps": 4,
96 + "num_stages": 3
97 },
98 "512": {
99 - "BLOCK_SIZE_M": 64,
100 - "BLOCK_SIZE_N": 256,
99 + "BLOCK_SIZE_M": 128,
100 + "BLOCK_SIZE_N": 128,
101 "BLOCK_SIZE_K": 128,
102 "GROUP_SIZE_M": 64,
103 - "num_warps": 4,
104 - "num_stages": 3
103 + "num_warps": 8,
104 + "num_stages": 5
105 },
106 "1024": {
107 "BLOCK_SIZE_M": 128,
108 "BLOCK_SIZE_N": 256,
109 "BLOCK_SIZE_K": 128,
110 - "GROUP_SIZE_M": 32,
110 + "GROUP_SIZE_M": 64,
111 "num_warps": 8,
112 "num_stages": 4
113 },
@@ -139,7 +139,7 @@
139 "BLOCK_SIZE_M": 128,
140 "BLOCK_SIZE_N": 256,
141 "BLOCK_SIZE_K": 128,
142 - "GROUP_SIZE_M": 32,
142 + "GROUP_SIZE_M": 16,
143 "num_warps": 8,
144 "num_stages": 4
145 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
+36 -36
@@ -1,17 +1,17 @@
1 {
2 "1": {
3 - "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 32,
5 - "BLOCK_SIZE_K": 64,
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 "GROUP_SIZE_M": 1,
7 "num_warps": 4,
8 "num_stages": 5
9 },
10 "2": {
11 "BLOCK_SIZE_M": 64,
12 - "BLOCK_SIZE_N": 64,
13 - "BLOCK_SIZE_K": 64,
14 - "GROUP_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 64,
15 "num_warps": 4,
16 "num_stages": 4
17 },
@@ -20,59 +20,59 @@
20 "BLOCK_SIZE_N": 128,
21 "BLOCK_SIZE_K": 128,
22 "GROUP_SIZE_M": 64,
23 - "num_warps": 8,
24 - "num_stages": 2
23 + "num_warps": 4,
24 + "num_stages": 3
25 },
26 "8": {
27 "BLOCK_SIZE_M": 64,
28 - "BLOCK_SIZE_N": 256,
28 + "BLOCK_SIZE_N": 64,
29 "BLOCK_SIZE_K": 128,
30 - "GROUP_SIZE_M": 32,
30 + "GROUP_SIZE_M": 1,
31 "num_warps": 4,
32 - "num_stages": 3
32 + "num_stages": 5
33 },
34 "16": {
35 "BLOCK_SIZE_M": 64,
36 - "BLOCK_SIZE_N": 128,
37 - "BLOCK_SIZE_K": 256,
38 - "GROUP_SIZE_M": 1,
39 - "num_warps": 8,
40 - "num_stages": 2
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 16,
39 + "num_warps": 4,
40 + "num_stages": 5
41 },
42 "24": {
43 "BLOCK_SIZE_M": 64,
44 - "BLOCK_SIZE_N": 128,
45 - "BLOCK_SIZE_K": 256,
46 - "GROUP_SIZE_M": 1,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 16,
47 "num_warps": 4,
48 - "num_stages": 3
48 + "num_stages": 5
49 },
50 "32": {
51 "BLOCK_SIZE_M": 64,
52 - "BLOCK_SIZE_N": 64,
53 - "BLOCK_SIZE_K": 128,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 256,
54 "GROUP_SIZE_M": 1,
55 - "num_warps": 8,
56 - "num_stages": 3
55 + "num_warps": 4,
56 + "num_stages": 4
57 },
58 "48": {
59 "BLOCK_SIZE_M": 64,
60 - "BLOCK_SIZE_N": 128,
60 + "BLOCK_SIZE_N": 64,
61 "BLOCK_SIZE_K": 128,
62 - "GROUP_SIZE_M": 64,
62 + "GROUP_SIZE_M": 1,
63 "num_warps": 4,
64 - "num_stages": 2
64 + "num_stages": 3
65 },
66 "64": {
67 "BLOCK_SIZE_M": 64,
68 "BLOCK_SIZE_N": 64,
69 - "BLOCK_SIZE_K": 256,
70 - "GROUP_SIZE_M": 16,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 "num_warps": 4,
72 - "num_stages": 2
72 + "num_stages": 3
73 },
74 "96": {
75 - "BLOCK_SIZE_M": 128,
75 + "BLOCK_SIZE_M": 64,
76 "BLOCK_SIZE_N": 64,
77 "BLOCK_SIZE_K": 128,
78 "GROUP_SIZE_M": 1,
@@ -81,11 +81,11 @@
81 },
82 "128": {
83 "BLOCK_SIZE_M": 64,
84 - "BLOCK_SIZE_N": 128,
85 - "BLOCK_SIZE_K": 256,
84 + "BLOCK_SIZE_N": 64,
85 + "BLOCK_SIZE_K": 128,
86 "GROUP_SIZE_M": 1,
87 - "num_warps": 8,
88 - "num_stages": 2
87 + "num_warps": 4,
88 + "num_stages": 3
89 },
90 "256": {
91 "BLOCK_SIZE_M": 128,
@@ -93,7 +93,7 @@
93 "BLOCK_SIZE_K": 128,
94 "GROUP_SIZE_M": 1,
95 "num_warps": 8,
96 - "num_stages": 5
96 + "num_stages": 3
97 },
98 "512": {
99 "BLOCK_SIZE_M": 128,