A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add unit test for Mixtral MoE layer (#2677)
Philipp Moritz committed
Jan 31, 2024 at 14:34 UTC
d0d93b92b190f420e2628350ec69921bede691d4
5 files changed
+119
-55
Dockerfile
+6
@@ -7,6 +7,12 @@ FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
7
RUN apt-get update -y \
8
&& apt-get install -y python3-pip git
9
10
+# Workaround for https://github.com/openai/triton/issues/2507 and
11
+# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
12
+# this won't be needed for future versions of this docker image
13
+# or future versions of triton.
14
+RUN ldconfig /usr/local/cuda-12.1/compat/
15
+
16
WORKDIR /workspace
17
18
# install build and runtime dependencies
tests/kernels/test_fused_moe.py
deleted
-50
@@ -1,50 +0,0 @@
1
-import pytest
2
-import torch
3
-
4
-from vllm.model_executor.layers.fused_moe import fused_moe
5
-from vllm.model_executor.layers.activation import SiluAndMul
6
-
7
-
8
-def torch_moe(a, w1, w2, topk_weight, topk_ids):
9
- B, D = a.shape
10
- a = a.view(B, -1, D).repeat(1, topk_ids.shape[1], 1).reshape(-1, D)
11
- out = torch.zeros(B * topk_ids.shape[1],
12
- w2.shape[1],
13
- dtype=a.dtype,
14
- device=a.device)
15
- topk_ids = topk_ids.view(-1)
16
- topk_weight = topk_weight.view(-1)
17
- for i in range(w1.shape[0]):
18
- mask = topk_ids == i
19
- if mask.sum():
20
- out[mask] = SiluAndMul()(
21
- a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
22
- return (out.view(B, -1, w2.shape[1]) *
23
- topk_weight.view(B, -1, 1)).sum(dim=1)
24
-
25
-
26
-@pytest.mark.parametrize("m", [512, 222, 33, 1])
27
-@pytest.mark.parametrize("n", [2048, 256, 1024])
28
-@pytest.mark.parametrize("k", [128, 511, 1024])
29
-@pytest.mark.parametrize("e", [8, 64])
30
-@pytest.mark.parametrize("topk", [2, 6])
31
-@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
32
-def test_fused_moe(
33
- m: int,
34
- n: int,
35
- k: int,
36
- e: int,
37
- topk: int,
38
- dtype: torch.dtype,
39
-):
40
- a = torch.randn((m, k), device='cuda', dtype=dtype) / 10
41
- w1 = torch.randn((e, 2 * n, k), device='cuda', dtype=dtype) / 10
42
- w2 = torch.randn((e, k, n), device='cuda', dtype=dtype) / 10
43
-
44
- score = torch.randn((m, e), device='cuda', dtype=dtype)
45
- score = torch.softmax(score, dim=-1)
46
- topk_weight, topk_ids = torch.topk(score, topk)
47
-
48
- triton_output = fused_moe(a, w1, w2, topk_weight, topk_ids, False)
49
- torch_output = torch_moe(a, w1, w2, topk_weight, topk_ids)
50
- assert torch.allclose(triton_output, torch_output, atol=1e-2, rtol=0)
tests/kernels/test_moe.py
new
+104
@@ -0,0 +1,104 @@
1
+"""Tests for the MOE layers.
2
+
3
+Run `pytest tests/kernels/test_moe.py`.
4
+"""
5
+
6
+import pytest
7
+import torch
8
+
9
+from transformers import MixtralConfig
10
+from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
11
+
12
+from vllm.model_executor.layers.fused_moe import fused_moe
13
+from vllm.model_executor.layers.activation import SiluAndMul
14
+from vllm.model_executor.models.mixtral import MixtralMoE
15
+
16
+
17
+def torch_moe(a, w1, w2, topk_weight, topk_ids):
18
+ B, D = a.shape
19
+ a = a.view(B, -1, D).repeat(1, topk_ids.shape[1], 1).reshape(-1, D)
20
+ out = torch.zeros(B * topk_ids.shape[1],
21
+ w2.shape[1],
22
+ dtype=a.dtype,
23
+ device=a.device)
24
+ topk_ids = topk_ids.view(-1)
25
+ topk_weight = topk_weight.view(-1)
26
+ for i in range(w1.shape[0]):
27
+ mask = topk_ids == i
28
+ if mask.sum():
29
+ out[mask] = SiluAndMul()(
30
+ a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
31
+ return (out.view(B, -1, w2.shape[1]) *
32
+ topk_weight.view(B, -1, 1)).sum(dim=1)
33
+
34
+
35
+@pytest.mark.parametrize("m", [512, 222, 33, 1])
36
+@pytest.mark.parametrize("n", [2048, 256, 1024])
37
+@pytest.mark.parametrize("k", [128, 511, 1024])
38
+@pytest.mark.parametrize("e", [8, 64])
39
+@pytest.mark.parametrize("topk", [2, 6])
40
+@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
41
+def test_fused_moe(
42
+ m: int,
43
+ n: int,
44
+ k: int,
45
+ e: int,
46
+ topk: int,
47
+ dtype: torch.dtype,
48
+):
49
+ a = torch.randn((m, k), device='cuda', dtype=dtype) / 10
50
+ w1 = torch.randn((e, 2 * n, k), device='cuda', dtype=dtype) / 10
51
+ w2 = torch.randn((e, k, n), device='cuda', dtype=dtype) / 10
52
+
53
+ score = torch.randn((m, e), device='cuda', dtype=dtype)
54
+ score = torch.softmax(score, dim=-1)
55
+ topk_weight, topk_ids = torch.topk(score, topk)
56
+
57
+ triton_output = fused_moe(a, w1, w2, topk_weight, topk_ids, False)
58
+ torch_output = torch_moe(a, w1, w2, topk_weight, topk_ids)
59
+ assert torch.allclose(triton_output, torch_output, atol=1e-2, rtol=0)
60
+
61
+
62
+@pytest.mark.parametrize("dtype",
63
+ [torch.float32, torch.float16, torch.bfloat16])
64
+@torch.inference_mode()
65
+def test_mixtral_moe(dtype: torch.dtype):
66
+ "Make sure our Mixtral MoE implementation agrees with the one from huggingface."
67
+
68
+ # Instantiate our and huggingface's MoE blocks
69
+ config = MixtralConfig()
70
+ hf_moe = MixtralSparseMoeBlock(config).to(dtype).to("cuda")
71
+ vllm_moe = MixtralMoE(
72
+ num_experts=config.num_local_experts,
73
+ top_k=config.num_experts_per_tok,
74
+ hidden_size=config.hidden_size,
75
+ intermediate_size=config.intermediate_size,
76
+ params_dtype=dtype,
77
+ tp_size=1,
78
+ )
79
+
80
+ # Load the weights
81
+ vllm_moe.gate.linear_weights["weight"][:] = hf_moe.gate.weight.data
82
+ for i in range(config.num_local_experts):
83
+ weights = (hf_moe.experts[i].w1.weight.data,
84
+ hf_moe.experts[i].w3.weight.data)
85
+ vllm_moe.ws[i][:] = torch.cat(weights, dim=0)
86
+ vllm_moe.w2s[i][:] = hf_moe.experts[i].w2.weight.data
87
+
88
+ # Generate input batch of dimensions [batch_size, seq_len, hidden_dim]
89
+ inputs = torch.randn((1, 64, config.hidden_size)).to(dtype).to("cuda")
90
+
91
+ # Run forward passes for both MoE blocks
92
+ hf_states, _ = hf_moe.forward(inputs)
93
+ vllm_states = vllm_moe.forward(inputs)
94
+
95
+ mixtral_moe_tol = {
96
+ torch.float32: 1e-3,
97
+ torch.float16: 1e-3,
98
+ torch.bfloat16: 1e-2,
99
+ }
100
+
101
+ assert torch.allclose(hf_states,
102
+ vllm_states,
103
+ rtol=mixtral_moe_tol[dtype],
104
+ atol=mixtral_moe_tol[dtype])
vllm/model_executor/layers/fused_moe.py
+3
-1
@@ -235,7 +235,9 @@ def fused_moe(hidden_states: torch.Tensor,
235
assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
236
assert w1.is_contiguous(), "Expert weights1 must be contiguous"
237
assert w2.is_contiguous(), "Expert weights2 must be contiguous"
238
- assert hidden_states.dtype in [torch.float16, torch.bfloat16]
238
+ assert hidden_states.dtype in [
239
+ torch.float32, torch.float16, torch.bfloat16
240
+ ]
241
M, _ = hidden_states.shape
242
E, N, _ = w1.shape
243
vllm/model_executor/models/mixtral.py
+6
-4
@@ -70,13 +70,14 @@ class MixtralMoE(nn.Module):
70
hidden_size: int,
71
intermediate_size: int,
72
params_dtype: Optional[torch.dtype] = None,
73
+ tp_size: Optional[int] = None,
74
):
75
super().__init__()
75
- tp_size = get_tensor_model_parallel_world_size()
76
+ self.tp_size = tp_size or get_tensor_model_parallel_world_size()
77
self.num_total_experts = num_experts
78
self.top_k = top_k
79
self.hidden_size = hidden_size
79
- self.intermediate_size = intermediate_size // tp_size
80
+ self.intermediate_size = intermediate_size // self.tp_size
81
82
if params_dtype is None:
83
params_dtype = torch.get_default_dtype()
@@ -141,8 +142,9 @@ class MixtralMoE(nn.Module):
142
selected_experts,
143
inplace=True)
144
144
- final_hidden_states = tensor_model_parallel_all_reduce(
145
- final_hidden_states)
145
+ if self.tp_size > 1:
146
+ final_hidden_states = tensor_model_parallel_all_reduce(
147
+ final_hidden_states)
148
149
return final_hidden_states.view(batch_size, sequence_length,
150
hidden_size)