@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51a08e7d8f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Re-tune Mixtral MoE configurations for FP8 on H100 (#5238)

Philipp Moritz committed Jun 5, 2024 at 10:59 UTC 51a08e7d8f0f11411d380c007ab606fc2d5e3cf9
3 files changed +85 -84
benchmarks/kernels/benchmark_moe.py
+2 -1
@@ -255,7 +255,8 @@ def main(args: argparse.Namespace):
255
256 if args.batch_size is None:
257 batch_sizes = [
258 - 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1536, 2048, 3072, 4096
258 + 1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
259 + 2048, 3072, 4096
260 ]
261 else:
262 batch_sizes = [args.batch_size]
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
+52 -52
@@ -1,113 +1,113 @@
1 {
2 "1": {
3 - "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 32,
5 - "BLOCK_SIZE_K": 64,
6 - "GROUP_SIZE_M": 1,
7 - "num_warps": 8,
8 - "num_stages": 4
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 4,
8 + "num_stages": 5
9 },
10 "2": {
11 - "BLOCK_SIZE_M": 16,
12 - "BLOCK_SIZE_N": 256,
13 - "BLOCK_SIZE_K": 64,
11 + "BLOCK_SIZE_M": 64,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 "GROUP_SIZE_M": 1,
15 - "num_warps": 8,
16 - "num_stages": 3
15 + "num_warps": 4,
16 + "num_stages": 4
17 },
18 "4": {
19 - "BLOCK_SIZE_M": 256,
20 - "BLOCK_SIZE_N": 64,
21 - "BLOCK_SIZE_K": 128,
22 - "GROUP_SIZE_M": 1,
19 + "BLOCK_SIZE_M": 64,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 16,
23 "num_warps": 4,
24 - "num_stages": 2
24 + "num_stages": 4
25 },
26 "8": {
27 "BLOCK_SIZE_M": 64,
28 - "BLOCK_SIZE_N": 256,
29 - "BLOCK_SIZE_K": 64,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 "GROUP_SIZE_M": 32,
31 - "num_warps": 8,
32 - "num_stages": 2
31 + "num_warps": 4,
32 + "num_stages": 4
33 },
34 "16": {
35 - "BLOCK_SIZE_M": 256,
36 - "BLOCK_SIZE_N": 128,
37 - "BLOCK_SIZE_K": 64,
35 + "BLOCK_SIZE_M": 64,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 "GROUP_SIZE_M": 1,
39 - "num_warps": 8,
40 - "num_stages": 5
39 + "num_warps": 4,
40 + "num_stages": 3
41 },
42 "24": {
43 "BLOCK_SIZE_M": 64,
44 - "BLOCK_SIZE_N": 256,
44 + "BLOCK_SIZE_N": 64,
45 "BLOCK_SIZE_K": 128,
46 "GROUP_SIZE_M": 64,
47 "num_warps": 4,
48 - "num_stages": 4
48 + "num_stages": 3
49 },
50 "32": {
51 "BLOCK_SIZE_M": 64,
52 - "BLOCK_SIZE_N": 128,
53 - "BLOCK_SIZE_K": 256,
54 - "GROUP_SIZE_M": 1,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 "num_warps": 4,
56 - "num_stages": 4
56 + "num_stages": 3
57 },
58 "48": {
59 "BLOCK_SIZE_M": 64,
60 - "BLOCK_SIZE_N": 256,
61 - "BLOCK_SIZE_K": 64,
62 - "GROUP_SIZE_M": 64,
60 + "BLOCK_SIZE_N": 64,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 16,
63 "num_warps": 4,
64 - "num_stages": 4
64 + "num_stages": 3
65 },
66 "64": {
67 "BLOCK_SIZE_M": 64,
68 - "BLOCK_SIZE_N": 128,
69 - "BLOCK_SIZE_K": 256,
68 + "BLOCK_SIZE_N": 64,
69 + "BLOCK_SIZE_K": 128,
70 "GROUP_SIZE_M": 1,
71 "num_warps": 4,
72 - "num_stages": 2
72 + "num_stages": 3
73 },
74 "96": {
75 - "BLOCK_SIZE_M": 128,
76 - "BLOCK_SIZE_N": 32,
75 + "BLOCK_SIZE_M": 64,
76 + "BLOCK_SIZE_N": 64,
77 "BLOCK_SIZE_K": 128,
78 "GROUP_SIZE_M": 1,
79 "num_warps": 4,
80 - "num_stages": 5
80 + "num_stages": 3
81 },
82 "128": {
83 "BLOCK_SIZE_M": 64,
84 "BLOCK_SIZE_N": 64,
85 - "BLOCK_SIZE_K": 256,
86 - "GROUP_SIZE_M": 32,
87 - "num_warps": 8,
88 - "num_stages": 5
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 3
89 },
90 "256": {
91 "BLOCK_SIZE_M": 64,
92 "BLOCK_SIZE_N": 128,
93 - "BLOCK_SIZE_K": 256,
94 - "GROUP_SIZE_M": 1,
95 - "num_warps": 8,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 16,
95 + "num_warps": 4,
96 "num_stages": 3
97 },
98 "512": {
99 "BLOCK_SIZE_M": 128,
100 - "BLOCK_SIZE_N": 64,
100 + "BLOCK_SIZE_N": 256,
101 "BLOCK_SIZE_K": 128,
102 "GROUP_SIZE_M": 16,
103 "num_warps": 8,
104 - "num_stages": 3
104 + "num_stages": 4
105 },
106 "1024": {
107 "BLOCK_SIZE_M": 128,
108 "BLOCK_SIZE_N": 256,
109 "BLOCK_SIZE_K": 128,
110 - "GROUP_SIZE_M": 64,
110 + "GROUP_SIZE_M": 32,
111 "num_warps": 8,
112 "num_stages": 4
113 },
@@ -115,7 +115,7 @@
115 "BLOCK_SIZE_M": 128,
116 "BLOCK_SIZE_N": 256,
117 "BLOCK_SIZE_K": 128,
118 - "GROUP_SIZE_M": 64,
118 + "GROUP_SIZE_M": 16,
119 "num_warps": 8,
120 "num_stages": 4
121 },
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=float8.json
+31 -31
@@ -1,67 +1,67 @@
1 {
2 "1": {
3 - "BLOCK_SIZE_M": 16,
4 - "BLOCK_SIZE_N": 32,
5 - "BLOCK_SIZE_K": 64,
6 - "GROUP_SIZE_M": 1,
3 + "BLOCK_SIZE_M": 64,
4 + "BLOCK_SIZE_N": 128,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 64,
7 "num_warps": 4,
8 - "num_stages": 5
8 + "num_stages": 4
9 },
10 "2": {
11 "BLOCK_SIZE_M": 64,
12 "BLOCK_SIZE_N": 64,
13 - "BLOCK_SIZE_K": 128,
14 - "GROUP_SIZE_M": 32,
15 - "num_warps": 8,
16 - "num_stages": 4
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 5
17 },
18 "4": {
19 "BLOCK_SIZE_M": 64,
20 - "BLOCK_SIZE_N": 128,
20 + "BLOCK_SIZE_N": 64,
21 "BLOCK_SIZE_K": 256,
22 "GROUP_SIZE_M": 1,
23 "num_warps": 4,
24 - "num_stages": 2
24 + "num_stages": 5
25 },
26 "8": {
27 - "BLOCK_SIZE_M": 128,
28 - "BLOCK_SIZE_N": 128,
27 + "BLOCK_SIZE_M": 64,
28 + "BLOCK_SIZE_N": 256,
29 "BLOCK_SIZE_K": 128,
30 - "GROUP_SIZE_M": 1,
30 + "GROUP_SIZE_M": 32,
31 "num_warps": 4,
32 - "num_stages": 3
32 + "num_stages": 4
33 },
34 "16": {
35 "BLOCK_SIZE_M": 64,
36 "BLOCK_SIZE_N": 128,
37 - "BLOCK_SIZE_K": 256,
38 - "GROUP_SIZE_M": 1,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 32,
39 "num_warps": 4,
40 - "num_stages": 4
40 + "num_stages": 5
41 },
42 "24": {
43 "BLOCK_SIZE_M": 64,
44 - "BLOCK_SIZE_N": 128,
45 - "BLOCK_SIZE_K": 128,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 256,
46 "GROUP_SIZE_M": 1,
47 "num_warps": 4,
48 - "num_stages": 5
48 + "num_stages": 3
49 },
50 "32": {
51 "BLOCK_SIZE_M": 64,
52 "BLOCK_SIZE_N": 128,
53 "BLOCK_SIZE_K": 256,
54 - "GROUP_SIZE_M": 64,
54 + "GROUP_SIZE_M": 1,
55 "num_warps": 4,
56 "num_stages": 4
57 },
58 "48": {
59 "BLOCK_SIZE_M": 64,
60 - "BLOCK_SIZE_N": 64,
60 + "BLOCK_SIZE_N": 128,
61 "BLOCK_SIZE_K": 256,
62 "GROUP_SIZE_M": 1,
63 "num_warps": 4,
64 - "num_stages": 3
64 + "num_stages": 4
65 },
66 "64": {
67 "BLOCK_SIZE_M": 64,
@@ -81,19 +81,19 @@
81 },
82 "128": {
83 "BLOCK_SIZE_M": 64,
84 - "BLOCK_SIZE_N": 64,
84 + "BLOCK_SIZE_N": 128,
85 "BLOCK_SIZE_K": 256,
86 "GROUP_SIZE_M": 1,
87 "num_warps": 4,
88 - "num_stages": 3
88 + "num_stages": 4
89 },
90 "256": {
91 - "BLOCK_SIZE_M": 128,
91 + "BLOCK_SIZE_M": 64,
92 "BLOCK_SIZE_N": 128,
93 "BLOCK_SIZE_K": 128,
94 - "GROUP_SIZE_M": 1,
95 - "num_warps": 8,
96 - "num_stages": 5
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 3
97 },
98 "512": {
99 "BLOCK_SIZE_M": 128,
@@ -107,7 +107,7 @@
107 "BLOCK_SIZE_M": 128,
108 "BLOCK_SIZE_N": 256,
109 "BLOCK_SIZE_K": 128,
110 - "GROUP_SIZE_M": 64,
110 + "GROUP_SIZE_M": 32,
111 "num_warps": 8,
112 "num_stages": 4
113 },