@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8fe8386591
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Kernel] change benchmark script so that result can be directly used; tune moe kernel in A100/H100 with tp=2,4,8 (#3389)

youkaichao committed Mar 14, 2024 at 01:11 UTC 8fe838659164b415d7f3044ec6b7e5bc52c6b6a5
9 files changed +903 -55
benchmarks/kernels/benchmark_mixtral_moe.py
+19 -11
@@ -2,13 +2,13 @@ import json
2 import os
3 import sys
4
5 -os.environ['CUDA_VISIBLE_DEVICES'] = '0'
6 -
7 -from vllm.model_executor.layers.fused_moe import fused_moe
5 +from vllm.model_executor.layers.fused_moe import fused_moe, get_config_file_name
6 import torch
7 import torch.nn.functional as F
8 import triton
9
10 +os.environ['CUDA_VISIBLE_DEVICES'] = '0'
11 +
12
13 def main():
14 method = fused_moe
@@ -64,7 +64,7 @@ def run_grid(bs, method):
64 print(f'{tp_size=} {bs=}')
65 print(f'{config}')
66 # warmup
67 - print(f'warming up')
67 + print('warming up')
68 try:
69 for _ in range(num_warmup_trials):
70 run_timing(
@@ -82,7 +82,7 @@ def run_grid(bs, method):
82 continue
83
84 # trial
85 - print(f'benchmarking')
85 + print('benchmarking')
86 for _ in range(num_trials):
87 kernel_dur_ms = run_timing(
88 num_calls=num_calls,
@@ -103,17 +103,25 @@ def run_grid(bs, method):
103 best_config = config
104 best_time_us = kernel_dur_us
105
106 - print(
107 - f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f} {bs=} {tp_size=} {top_k=} {num_total_experts=} {d_model=} {model_intermediate_size=} {num_layers=}'
108 - )
106 + print(f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f}'
107 + f' {bs=} {tp_size=} {top_k=} {num_total_experts=} '
108 + f'{d_model=} {model_intermediate_size=} {num_layers=}')
109
110 print("best_time_us", best_time_us)
111 print("best_config", best_config)
112
113 - filename = "/tmp/config.jsonl"
113 + # holds Dict[str, Dict[str, int]]
114 + filename = get_config_file_name(num_total_experts,
115 + model_intermediate_size // tp_size)
116 print(f"writing config to file {filename}")
115 - with open(filename, "a") as f:
116 - f.write(json.dumps({str(bs): best_config}) + "\n")
117 + existing_content = {}
118 + if os.path.exists(filename):
119 + with open(filename, "r") as f:
120 + existing_content = json.load(f)
121 + existing_content[str(bs)] = best_config
122 + with open(filename, "w") as f:
123 + json.dump(existing_content, f, indent=4)
124 + f.write("\n")
125
126
127 def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
vllm/model_executor/layers/fused_moe/__init__.py
+5 -1
@@ -1,5 +1,9 @@
1 -from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
1 +from vllm.model_executor.layers.fused_moe.fused_moe import (
2 + fused_moe,
3 + get_config_file_name,
4 +)
5
6 __all__ = [
7 "fused_moe",
8 + "get_config_file_name",
9 ]
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 16,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 128,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 1,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 32,
39 + "num_warps": 8,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 32,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 32,
47 + "num_warps": 8,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 32,
53 + "BLOCK_SIZE_K": 256,
54 + "GROUP_SIZE_M": 32,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 32,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 32,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 32,
69 + "BLOCK_SIZE_K": 256,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 32,
77 + "BLOCK_SIZE_K": 64,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 16,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 64,
87 + "num_warps": 8,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 64,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 32,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 64,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 32,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 8,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 64,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 256,
37 + "BLOCK_SIZE_K": 64,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 8,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 32,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 64,
46 + "GROUP_SIZE_M": 64,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 32,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 64,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 64,
70 + "GROUP_SIZE_M": 16,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 1,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json
+144 -18
@@ -1,20 +1,146 @@
1 {
2 - "1": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
3 - "2": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
4 - "4": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
5 - "8": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
6 - "16": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
7 - "24": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
8 - "32": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
9 - "64": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
10 - "96": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
11 - "128": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
12 - "192": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
13 - "256": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4},
14 - "512": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 32, "num_warps": 8, "num_stages": 4},
15 - "1024": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
16 - "1536": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4},
17 - "2048": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
18 - "3072": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 8, "num_stages": 4},
19 - "4096": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4}
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 32,
7 + "num_warps": 8,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 1,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 128,
22 + "GROUP_SIZE_M": 32,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 64,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 16,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 64,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 16,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 32,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 16,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 32,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 }
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 256,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 256,
13 + "BLOCK_SIZE_K": 64,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 1,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 32,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 128,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 1,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 128,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 4,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 64,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 1,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json new
+146
@@ -0,0 +1,146 @@
1 +{
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 64,
6 + "GROUP_SIZE_M": 1,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 64,
13 + "BLOCK_SIZE_K": 256,
14 + "GROUP_SIZE_M": 16,
15 + "num_warps": 4,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 64,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 64,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 64,
29 + "BLOCK_SIZE_K": 256,
30 + "GROUP_SIZE_M": 16,
31 + "num_warps": 4,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 64,
37 + "BLOCK_SIZE_K": 256,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 64,
45 + "BLOCK_SIZE_K": 256,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 64,
53 + "BLOCK_SIZE_K": 256,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 16,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 32,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 32,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 256,
93 + "BLOCK_SIZE_K": 64,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 8,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 8,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 64,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 64,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 64,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 32,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 128,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 16,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 128,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 16,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 128,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 +}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json
+144 -22
@@ -1,24 +1,146 @@
1 {
2 - "1": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
3 - "2": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 256, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
4 - "4": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 256, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
5 - "8": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 8, "num_stages": 4},
6 - "16": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
7 - "24": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
8 - "32": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
9 - "80": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
10 - "96": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
11 - "128": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
12 - "192": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
13 - "200": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 4, "num_stages": 4},
14 - "208": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 4, "num_stages": 4},
15 - "216": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
16 - "224": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 32, "num_warps": 4, "num_stages": 4},
17 - "256": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 32, "num_warps": 4, "num_stages": 4},
18 - "512": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
19 - "1024": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
20 - "1536": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
21 - "2048": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
22 - "3072": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
23 - "4096": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4}
2 + "1": {
3 + "BLOCK_SIZE_M": 16,
4 + "BLOCK_SIZE_N": 64,
5 + "BLOCK_SIZE_K": 128,
6 + "GROUP_SIZE_M": 64,
7 + "num_warps": 4,
8 + "num_stages": 4
9 + },
10 + "2": {
11 + "BLOCK_SIZE_M": 16,
12 + "BLOCK_SIZE_N": 128,
13 + "BLOCK_SIZE_K": 128,
14 + "GROUP_SIZE_M": 32,
15 + "num_warps": 8,
16 + "num_stages": 4
17 + },
18 + "4": {
19 + "BLOCK_SIZE_M": 16,
20 + "BLOCK_SIZE_N": 32,
21 + "BLOCK_SIZE_K": 256,
22 + "GROUP_SIZE_M": 16,
23 + "num_warps": 4,
24 + "num_stages": 4
25 + },
26 + "8": {
27 + "BLOCK_SIZE_M": 16,
28 + "BLOCK_SIZE_N": 128,
29 + "BLOCK_SIZE_K": 128,
30 + "GROUP_SIZE_M": 32,
31 + "num_warps": 8,
32 + "num_stages": 4
33 + },
34 + "16": {
35 + "BLOCK_SIZE_M": 16,
36 + "BLOCK_SIZE_N": 128,
37 + "BLOCK_SIZE_K": 128,
38 + "GROUP_SIZE_M": 1,
39 + "num_warps": 4,
40 + "num_stages": 4
41 + },
42 + "24": {
43 + "BLOCK_SIZE_M": 16,
44 + "BLOCK_SIZE_N": 128,
45 + "BLOCK_SIZE_K": 128,
46 + "GROUP_SIZE_M": 1,
47 + "num_warps": 4,
48 + "num_stages": 4
49 + },
50 + "32": {
51 + "BLOCK_SIZE_M": 16,
52 + "BLOCK_SIZE_N": 128,
53 + "BLOCK_SIZE_K": 128,
54 + "GROUP_SIZE_M": 1,
55 + "num_warps": 4,
56 + "num_stages": 4
57 + },
58 + "48": {
59 + "BLOCK_SIZE_M": 32,
60 + "BLOCK_SIZE_N": 128,
61 + "BLOCK_SIZE_K": 128,
62 + "GROUP_SIZE_M": 1,
63 + "num_warps": 4,
64 + "num_stages": 4
65 + },
66 + "64": {
67 + "BLOCK_SIZE_M": 32,
68 + "BLOCK_SIZE_N": 128,
69 + "BLOCK_SIZE_K": 128,
70 + "GROUP_SIZE_M": 1,
71 + "num_warps": 4,
72 + "num_stages": 4
73 + },
74 + "96": {
75 + "BLOCK_SIZE_M": 32,
76 + "BLOCK_SIZE_N": 128,
77 + "BLOCK_SIZE_K": 128,
78 + "GROUP_SIZE_M": 64,
79 + "num_warps": 4,
80 + "num_stages": 4
81 + },
82 + "128": {
83 + "BLOCK_SIZE_M": 64,
84 + "BLOCK_SIZE_N": 128,
85 + "BLOCK_SIZE_K": 128,
86 + "GROUP_SIZE_M": 1,
87 + "num_warps": 4,
88 + "num_stages": 4
89 + },
90 + "256": {
91 + "BLOCK_SIZE_M": 64,
92 + "BLOCK_SIZE_N": 128,
93 + "BLOCK_SIZE_K": 128,
94 + "GROUP_SIZE_M": 64,
95 + "num_warps": 4,
96 + "num_stages": 4
97 + },
98 + "512": {
99 + "BLOCK_SIZE_M": 64,
100 + "BLOCK_SIZE_N": 256,
101 + "BLOCK_SIZE_K": 64,
102 + "GROUP_SIZE_M": 32,
103 + "num_warps": 4,
104 + "num_stages": 4
105 + },
106 + "1024": {
107 + "BLOCK_SIZE_M": 128,
108 + "BLOCK_SIZE_N": 256,
109 + "BLOCK_SIZE_K": 64,
110 + "GROUP_SIZE_M": 16,
111 + "num_warps": 8,
112 + "num_stages": 4
113 + },
114 + "1536": {
115 + "BLOCK_SIZE_M": 128,
116 + "BLOCK_SIZE_N": 256,
117 + "BLOCK_SIZE_K": 64,
118 + "GROUP_SIZE_M": 64,
119 + "num_warps": 8,
120 + "num_stages": 4
121 + },
122 + "2048": {
123 + "BLOCK_SIZE_M": 128,
124 + "BLOCK_SIZE_N": 256,
125 + "BLOCK_SIZE_K": 64,
126 + "GROUP_SIZE_M": 32,
127 + "num_warps": 8,
128 + "num_stages": 4
129 + },
130 + "3072": {
131 + "BLOCK_SIZE_M": 128,
132 + "BLOCK_SIZE_N": 256,
133 + "BLOCK_SIZE_K": 64,
134 + "GROUP_SIZE_M": 64,
135 + "num_warps": 8,
136 + "num_stages": 4
137 + },
138 + "4096": {
139 + "BLOCK_SIZE_M": 128,
140 + "BLOCK_SIZE_N": 256,
141 + "BLOCK_SIZE_K": 64,
142 + "GROUP_SIZE_M": 16,
143 + "num_warps": 8,
144 + "num_stages": 4
145 + }
146 }
vllm/model_executor/layers/fused_moe/fused_moe.py
+7 -3
@@ -245,6 +245,11 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
245 )
246
247
248 +def get_config_file_name(E: int, N: int) -> str:
249 + device_name = torch.cuda.get_device_name().replace(" ", "_")
250 + return f"E={E},N={N},device_name={device_name}.json"
251 +
252 +
253 @functools.lru_cache
254 def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
255 """
@@ -258,11 +263,10 @@ def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
263
264 # First look up if an optimized configuration is available in the configs
265 # directory
261 - device_name = torch.cuda.get_device_name().replace(" ", "_")
266 + json_file_name = get_config_file_name(E, N)
267
268 config_file_path = os.path.join(
264 - os.path.dirname(os.path.realpath(__file__)), "configs",
265 - f"E={E},N={N},device_name={device_name}.json")
269 + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
270 if os.path.exists(config_file_path):
271 with open(config_file_path) as f:
272 logger.info(