A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Optimize Triton MoE Kernel (#2979)
Co-authored-by: Cade Daniel <edacih@gmail.com>
Philipp Moritz committed
Feb 26, 2024 at 13:48 UTC
cfc15a1031ef0197a1b291d2ed93717a9bdad268
7 files changed
+297
-15
benchmarks/kernels/benchmark_mixtral_moe.py
new
+172
@@ -0,0 +1,172 @@
1
+import json
2
+import os
3
+import sys
4
+
5
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
6
+
7
+from vllm.model_executor.layers.fused_moe import fused_moe
8
+import torch
9
+import torch.nn.functional as F
10
+import triton
11
+
12
+
13
+def main():
14
+ method = fused_moe
15
+ for bs in [
16
+ 1, 2, 4, 8, 16, 24, 32, 48, 64, 96, 128, 256, 512, 1024, 1536,
17
+ 2048, 3072, 4096
18
+ ]:
19
+ run_grid(bs, method=method)
20
+
21
+
22
+def run_grid(bs, method):
23
+ d_model = 4096
24
+ num_total_experts = 8
25
+ top_k = 2
26
+ tp_size = 2
27
+ model_intermediate_size = 14336
28
+ num_layers = 32
29
+ num_calls = 100
30
+
31
+ num_warmup_trials = 1
32
+ num_trials = 1
33
+
34
+ configs = []
35
+ if bs <= 16:
36
+ BLOCK_SIZES_M = [16]
37
+ elif bs <= 32:
38
+ BLOCK_SIZES_M = [16, 32]
39
+ elif bs <= 64:
40
+ BLOCK_SIZES_M = [16, 32, 64]
41
+ elif bs <= 128:
42
+ BLOCK_SIZES_M = [16, 32, 64, 128]
43
+ else:
44
+ BLOCK_SIZES_M = [16, 32, 64, 128, 256]
45
+
46
+ for block_size_n in [32, 64, 128, 256]:
47
+ for block_size_m in BLOCK_SIZES_M:
48
+ for block_size_k in [64, 128, 256]:
49
+ for group_size_m in [1, 16, 32, 64]:
50
+ for num_warps in [4, 8]:
51
+ configs.append({
52
+ "BLOCK_SIZE_M": block_size_m,
53
+ "BLOCK_SIZE_N": block_size_n,
54
+ "BLOCK_SIZE_K": block_size_k,
55
+ "GROUP_SIZE_M": group_size_m,
56
+ "num_warps": num_warps,
57
+ "num_stages": 4,
58
+ })
59
+
60
+ best_config = None
61
+ best_time_us = 1e20
62
+
63
+ for config in configs:
64
+ print(f'{tp_size=} {bs=}')
65
+ print(f'{config}')
66
+ # warmup
67
+ print(f'warming up')
68
+ try:
69
+ for _ in range(num_warmup_trials):
70
+ run_timing(
71
+ num_calls=num_calls,
72
+ bs=bs,
73
+ d_model=d_model,
74
+ num_total_experts=num_total_experts,
75
+ top_k=top_k,
76
+ tp_size=tp_size,
77
+ model_intermediate_size=model_intermediate_size,
78
+ method=method,
79
+ config=config,
80
+ )
81
+ except triton.runtime.autotuner.OutOfResources:
82
+ continue
83
+
84
+ # trial
85
+ print(f'benchmarking')
86
+ for _ in range(num_trials):
87
+ kernel_dur_ms = run_timing(
88
+ num_calls=num_calls,
89
+ bs=bs,
90
+ d_model=d_model,
91
+ num_total_experts=num_total_experts,
92
+ top_k=top_k,
93
+ tp_size=tp_size,
94
+ model_intermediate_size=model_intermediate_size,
95
+ method=method,
96
+ config=config,
97
+ )
98
+
99
+ kernel_dur_us = 1000 * kernel_dur_ms
100
+ model_dur_ms = kernel_dur_ms * num_layers
101
+
102
+ if kernel_dur_us < best_time_us:
103
+ best_config = config
104
+ best_time_us = kernel_dur_us
105
+
106
+ print(
107
+ f'{kernel_dur_us=:.1f} {model_dur_ms=:.1f} {bs=} {tp_size=} {top_k=} {num_total_experts=} {d_model=} {model_intermediate_size=} {num_layers=}'
108
+ )
109
+
110
+ print("best_time_us", best_time_us)
111
+ print("best_config", best_config)
112
+
113
+ filename = "/tmp/config.jsonl"
114
+ print(f"writing config to file {filename}")
115
+ with open(filename, "a") as f:
116
+ f.write(json.dumps({str(bs): best_config}) + "\n")
117
+
118
+
119
+def run_timing(num_calls: int, bs: int, d_model: int, num_total_experts: int,
120
+ top_k: int, tp_size: int, model_intermediate_size: int, method,
121
+ config) -> float:
122
+ shard_intermediate_size = model_intermediate_size // tp_size
123
+
124
+ hidden_states = torch.rand(
125
+ (bs, d_model),
126
+ device="cuda:0",
127
+ dtype=torch.bfloat16,
128
+ )
129
+
130
+ ws = torch.rand(
131
+ (num_total_experts, 2 * shard_intermediate_size, d_model),
132
+ device=hidden_states.device,
133
+ dtype=hidden_states.dtype,
134
+ )
135
+
136
+ w2s = torch.rand(
137
+ (num_total_experts, d_model, shard_intermediate_size),
138
+ device=hidden_states.device,
139
+ dtype=hidden_states.dtype,
140
+ )
141
+
142
+ gating_output = F.softmax(torch.rand(
143
+ (num_calls, bs, num_total_experts),
144
+ device=hidden_states.device,
145
+ dtype=torch.float32,
146
+ ),
147
+ dim=-1)
148
+
149
+ start_event = torch.cuda.Event(enable_timing=True)
150
+ end_event = torch.cuda.Event(enable_timing=True)
151
+
152
+ start_event.record()
153
+ for i in range(num_calls):
154
+ hidden_states = method(
155
+ hidden_states=hidden_states,
156
+ w1=ws,
157
+ w2=w2s,
158
+ gating_output=gating_output[i],
159
+ topk=2,
160
+ renormalize=True,
161
+ inplace=True,
162
+ override_config=config,
163
+ )
164
+ end_event.record()
165
+ end_event.synchronize()
166
+
167
+ dur_ms = start_event.elapsed_time(end_event) / num_calls
168
+ return dur_ms
169
+
170
+
171
+if __name__ == "__main__":
172
+ sys.exit(main())
setup.py
+3
-1
@@ -432,7 +432,9 @@ def get_requirements() -> List[str]:
432
return requirements
433
434
435
-package_data = {"vllm": ["py.typed"]}
435
+package_data = {
436
+ "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
437
+}
438
if os.environ.get("VLLM_USE_PRECOMPILED"):
439
ext_modules = []
440
package_data["vllm"].append("*.so")
vllm/model_executor/layers/fused_moe/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
2
+
3
+__all__ = [
4
+ "fused_moe",
5
+]
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json
new
+20
@@ -0,0 +1,20 @@
1
+{
2
+ "1": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
3
+ "2": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
4
+ "4": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
5
+ "8": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
6
+ "16": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 7},
7
+ "24": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
8
+ "32": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
9
+ "64": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
10
+ "96": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
11
+ "128": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
12
+ "192": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 6},
13
+ "256": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4},
14
+ "512": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 32, "num_warps": 8, "num_stages": 4},
15
+ "1024": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
16
+ "1536": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4},
17
+ "2048": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
18
+ "3072": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 1, "num_warps": 8, "num_stages": 4},
19
+ "4096": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32, "GROUP_SIZE_M": 16, "num_warps": 8, "num_stages": 4}
20
+}
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json
new
+24
@@ -0,0 +1,24 @@
1
+{
2
+ "1": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
3
+ "2": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 256, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
4
+ "4": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 256, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
5
+ "8": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 8, "num_stages": 4},
6
+ "16": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
7
+ "24": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 4, "num_warps": 4, "num_stages": 4},
8
+ "32": {"BLOCK_SIZE_M": 16, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
9
+ "80": {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
10
+ "96": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
11
+ "128": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
12
+ "192": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 1, "num_warps": 4, "num_stages": 4},
13
+ "200": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 4, "num_stages": 4},
14
+ "208": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 2, "num_warps": 4, "num_stages": 4},
15
+ "216": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 16, "num_warps": 4, "num_stages": 4},
16
+ "224": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 128, "GROUP_SIZE_M": 32, "num_warps": 4, "num_stages": 4},
17
+ "256": {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 32, "num_warps": 4, "num_stages": 4},
18
+ "512": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
19
+ "1024": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
20
+ "1536": {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
21
+ "2048": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
22
+ "3072": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4},
23
+ "4096": {"BLOCK_SIZE_M": 256, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64, "GROUP_SIZE_M": 64, "num_warps": 8, "num_stages": 4}
24
+}
vllm/model_executor/layers/fused_moe/configs/README
new
+10
@@ -0,0 +1,10 @@
1
+This directory contains tuned configurations for different settings of the fused_moe kernel.
2
+For different settings of
3
+- E (number of experts)
4
+- N (intermediate size)
5
+- device_name (torch.cuda.get_device_name())
6
+the JSON file contains a mapping from M (batch size) to the chosen configuration.
7
+
8
+The example configurations provided are for the Mixtral model for TP2 on H100
9
+and TP4 on A100. Mixtral has intermediate size N = 14336, i.e. for TP2 we have
10
+N = 7168 and for TP4 we have N = 3584.
vllm/model_executor/layers/fused_moe/fused_moe.py
renamed
+63
-14
@@ -1,11 +1,19 @@
1
"""Fused MoE kernel."""
2
+import functools
3
+import json
4
+import os
5
+from typing import Any, Dict, Optional
6
+
7
import torch
8
import triton
9
import triton.language as tl
10
11
from vllm._C import ops
12
+from vllm.logger import init_logger
13
from vllm.utils import is_hip
14
15
+logger = init_logger(__name__)
16
+
17
18
@triton.jit
19
def fused_moe_kernel(
@@ -210,6 +218,34 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
218
)
219
220
221
+@functools.lru_cache
222
+def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
223
+ """
224
+ Return optimized configurations for the fused MoE kernel.
225
+
226
+ The return value will be a dictionary that maps an irregular grid of batch sizes
227
+ to configurations of the fused_moe kernel. To evaluate the kernel on a given batch
228
+ size bs, the closest batch size in the grid should be picked and the associated
229
+ configuration chosen to invoke the kernel.
230
+ """
231
+
232
+ # First look up if an optimized configuration is available in the configs directory
233
+ device_name = torch.cuda.get_device_name().replace(" ", "_")
234
+
235
+ config_file_path = os.path.join(
236
+ os.path.dirname(os.path.realpath(__file__)), "configs",
237
+ f"E={E},N={N},device_name={device_name}.json")
238
+ if os.path.exists(config_file_path):
239
+ with open(config_file_path) as f:
240
+ logger.info(
241
+ f"Using configuration from {config_file_path} for MoE layer.")
242
+ # If a configuration has been found, return it
243
+ return {int(key): val for key, val in json.load(f).items()}
244
+
245
+ # If no optimized configuration is available, we will use the default configuration
246
+ return None
247
+
248
+
249
def fused_moe(
250
hidden_states: torch.Tensor,
251
w1: torch.Tensor,
@@ -218,6 +254,7 @@ def fused_moe(
254
topk: int,
255
renormalize: bool,
256
inplace: bool = False,
257
+ override_config: Optional[Dict[str, Any]] = None,
258
) -> torch.Tensor:
259
"""
260
This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism.
@@ -230,6 +267,7 @@ def fused_moe(
267
- topk (int): The number of top-k experts to select.
268
- renormalize (bool): If True, renormalize the top-k weights to sum to 1.
269
- inplace (bool): If True, perform the operation in-place. Defaults to False.
270
+ - override_config (Optional[Dict[str, Any]]): Optional override for the kernel configuration.
271
272
Returns:
273
- torch.Tensor: The output tensor after applying the MoE layer.
@@ -279,20 +317,31 @@ def fused_moe(
317
if renormalize:
318
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
319
282
- config = {
283
- 'BLOCK_SIZE_M': 64,
284
- 'BLOCK_SIZE_N': 64,
285
- 'BLOCK_SIZE_K': 32,
286
- 'GROUP_SIZE_M': 8
287
- }
288
-
289
- if topk_ids.numel() <= w1.shape[0]:
290
- config = {
291
- 'BLOCK_SIZE_M': 16,
292
- 'BLOCK_SIZE_N': 32,
293
- 'BLOCK_SIZE_K': 64,
294
- 'GROUP_SIZE_M': 1
295
- }
320
+ if override_config:
321
+ config = override_config
322
+ else:
323
+ # First try to load optimal config from the file
324
+ configs = get_moe_configs(E, w2.shape[2])
325
+
326
+ if configs:
327
+ # If an optimal configuration map has been found, look up the optimal config
328
+ config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
329
+ else:
330
+ # Else use the default config
331
+ config = {
332
+ 'BLOCK_SIZE_M': 64,
333
+ 'BLOCK_SIZE_N': 64,
334
+ 'BLOCK_SIZE_K': 32,
335
+ 'GROUP_SIZE_M': 8
336
+ }
337
+
338
+ if M <= E:
339
+ config = {
340
+ 'BLOCK_SIZE_M': 16,
341
+ 'BLOCK_SIZE_N': 32,
342
+ 'BLOCK_SIZE_K': 64,
343
+ 'GROUP_SIZE_M': 1
344
+ }
345
346
intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N),
347
device=hidden_states.device,