A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
DeepseekMoE support with Fused MoE kernel (#2453)
Co-authored-by: roy <jasonailu87@gmail.com>
wangding zeng committed
Jan 30, 2024 at 13:19 UTC
5d60def02cb5a43fa5864fcb123909b101df9ec5
9 files changed
+924
csrc/dispatch_utils.h
+11
@@ -24,3 +24,14 @@
24
#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
25
AT_DISPATCH_SWITCH( \
26
TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
27
+
28
+#define VLLM_DISPATCH_CASE_INTEGRAL_TYPES(...) \
29
+ AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \
30
+ AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \
31
+ AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \
32
+ AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \
33
+ AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__)
34
+
35
+#define VLLM_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
36
+ AT_DISPATCH_SWITCH( \
37
+ TYPE, NAME, VLLM_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
csrc/moe_align_block_size_kernels.cu
new
+108
@@ -0,0 +1,108 @@
1
+#include <torch/extension.h>
2
+#include <ATen/cuda/CUDAContext.h>
3
+
4
+#include <ATen/ATen.h>
5
+#include <THC/THCAtomics.cuh>
6
+
7
+#include "cuda_compat.h"
8
+#include "dispatch_utils.h"
9
+
10
+const static size_t NUM_MAX_EXPERTS = 64;
11
+#define CEILDIV(x,y) (((x) + (y) - 1) / (y))
12
+
13
+namespace vllm {
14
+template <typename scalar_t>
15
+__global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
16
+ int32_t *sorted_token_ids,
17
+ int32_t *expert_ids,
18
+ int32_t *total_tokens_post_pad,
19
+ int32_t num_experts,
20
+ int32_t block_size,
21
+ size_t numel) {
22
+ const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
23
+ const size_t start_idx = threadIdx.x * tokens_per_thread;
24
+ __shared__ int32_t tokens_cnts[NUM_MAX_EXPERTS + 1][NUM_MAX_EXPERTS];
25
+ __shared__ int32_t cumsum[NUM_MAX_EXPERTS + 1];
26
+ for (int i = 0; i < num_experts; ++i) {
27
+ tokens_cnts[threadIdx.x + 1][i] = 0;
28
+ }
29
+
30
+ /**
31
+ * In the first step we compute token_cnts[thread_index + 1][expert_index],
32
+ * which counts how many tokens in the token shard of thread_index are assigned
33
+ * to expert expert_index.
34
+ */
35
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
36
+ ++tokens_cnts[threadIdx.x + 1][topk_ids[i]];
37
+ }
38
+
39
+ __syncthreads();
40
+
41
+ // For each expert we accumulate the token counts from the different threads.
42
+ tokens_cnts[0][threadIdx.x] = 0;
43
+ for (int i = 1; i <= blockDim.x; ++i) {
44
+ tokens_cnts[i][threadIdx.x] += tokens_cnts[i-1][threadIdx.x];
45
+ }
46
+
47
+ __syncthreads();
48
+
49
+ // We accumulate the token counts of all experts in thread 0.
50
+ if (threadIdx.x == 0) {
51
+ cumsum[0] = 0;
52
+ for (int i = 1; i <= num_experts; ++i) {
53
+ cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[blockDim.x][i - 1], block_size) * block_size;
54
+ }
55
+ *total_tokens_post_pad = cumsum[num_experts];
56
+ }
57
+
58
+ __syncthreads();
59
+
60
+ /**
61
+ * For each expert, each thread processes the tokens of the corresponding blocks
62
+ * and stores the corresponding expert_id for each block.
63
+ */
64
+ for (int i = cumsum[threadIdx.x];i < cumsum[threadIdx.x + 1];i += block_size) {
65
+ expert_ids[i / block_size] = threadIdx.x;
66
+ }
67
+
68
+ /**
69
+ * Each thread processes a token shard, calculating the index of each token after
70
+ * sorting by expert number. Given the example topk_ids = [0,1,2,1,2,3,0,3,4] and
71
+ * block_size = 4, then the output would be [0, 6, *, *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *],
72
+ * where * represents a padding value(preset in python).
73
+ */
74
+ for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
75
+ int32_t expert_id = topk_ids[i];
76
+ /** The cumsum[expert_id] stores the starting index of the tokens that the
77
+ * expert with expert_id needs to process, and tokens_cnts[threadIdx.x][expert_id]
78
+ * stores the indices of the tokens processed by the expert with expert_id within
79
+ * the current thread's token shard.
80
+ */
81
+ int32_t rank_post_pad = tokens_cnts[threadIdx.x][expert_id] + cumsum[expert_id];
82
+ sorted_token_ids[rank_post_pad] = i;
83
+ ++tokens_cnts[threadIdx.x][expert_id];
84
+ }
85
+}
86
+}
87
+
88
+void moe_align_block_size(
89
+ torch::Tensor topk_ids,
90
+ int num_experts,
91
+ int block_size,
92
+ torch::Tensor sorted_token_ids,
93
+ torch::Tensor experts_ids,
94
+ torch::Tensor num_tokens_post_pad) {
95
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
96
+ assert(num_experts <= NUM_MAX_EXPERTS);
97
+ VLLM_DISPATCH_INTEGRAL_TYPES(
98
+ topk_ids.scalar_type(), "moe_alig_block_size_kernel", [&] {
99
+ vllm::moe_align_block_size_kernel<scalar_t><<<1, num_experts, 0, stream>>>(
100
+ topk_ids.data_ptr<scalar_t>(),
101
+ sorted_token_ids.data_ptr<int32_t>(),
102
+ experts_ids.data_ptr<int32_t>(),
103
+ num_tokens_post_pad.data_ptr<int32_t>(),
104
+ num_experts,
105
+ block_size,
106
+ topk_ids.numel());
107
+ });
108
+}
csrc/ops.h
+9
@@ -121,3 +121,12 @@ std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
121
void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
122
const std::vector<std::vector<int64_t>> &offsets);
123
#endif
124
+
125
+void moe_align_block_size(
126
+ torch::Tensor topk_ids,
127
+ int num_experts,
128
+ int block_size,
129
+ torch::Tensor sorted_token_ids,
130
+ torch::Tensor experts_ids,
131
+ torch::Tensor num_tokens_post_pad
132
+ );
csrc/pybind.cpp
+4
@@ -56,6 +56,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
56
ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
57
ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
58
ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
59
+ ops.def(
60
+ "moe_align_block_size",
61
+ &moe_align_block_size,
62
+ "Aligning the number of tokens to be processed by each expert such that it is divisible by the block size.");
63
64
// Cache ops
65
pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
setup.py
+1
@@ -309,6 +309,7 @@ vllm_extension_sources = [
309
"csrc/quantization/squeezellm/quant_cuda_kernel.cu",
310
"csrc/quantization/gptq/q_gemm.cu",
311
"csrc/cuda_utils_kernels.cu",
312
+ "csrc/moe_align_block_size_kernels.cu",
313
"csrc/pybind.cpp",
314
]
315
tests/kernels/test_fused_moe.py
new
+50
@@ -0,0 +1,50 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm.model_executor.layers.fused_moe import fused_moe
5
+from vllm.model_executor.layers.activation import SiluAndMul
6
+
7
+
8
+def torch_moe(a, w1, w2, topk_weight, topk_ids):
9
+ B, D = a.shape
10
+ a = a.view(B, -1, D).repeat(1, topk_ids.shape[1], 1).reshape(-1, D)
11
+ out = torch.zeros(B * topk_ids.shape[1],
12
+ w2.shape[1],
13
+ dtype=a.dtype,
14
+ device=a.device)
15
+ topk_ids = topk_ids.view(-1)
16
+ topk_weight = topk_weight.view(-1)
17
+ for i in range(w1.shape[0]):
18
+ mask = topk_ids == i
19
+ if mask.sum():
20
+ out[mask] = SiluAndMul()(
21
+ a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
22
+ return (out.view(B, -1, w2.shape[1]) *
23
+ topk_weight.view(B, -1, 1)).sum(dim=1)
24
+
25
+
26
+@pytest.mark.parametrize("m", [512, 222, 33, 1])
27
+@pytest.mark.parametrize("n", [2048, 256, 1024])
28
+@pytest.mark.parametrize("k", [128, 511, 1024])
29
+@pytest.mark.parametrize("e", [8, 64])
30
+@pytest.mark.parametrize("topk", [2, 6])
31
+@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
32
+def test_fused_moe(
33
+ m: int,
34
+ n: int,
35
+ k: int,
36
+ e: int,
37
+ topk: int,
38
+ dtype: torch.dtype,
39
+):
40
+ a = torch.randn((m, k), device='cuda', dtype=dtype) / 10
41
+ w1 = torch.randn((e, 2 * n, k), device='cuda', dtype=dtype) / 10
42
+ w2 = torch.randn((e, k, n), device='cuda', dtype=dtype) / 10
43
+
44
+ score = torch.randn((m, e), device='cuda', dtype=dtype)
45
+ score = torch.softmax(score, dim=-1)
46
+ topk_weight, topk_ids = torch.topk(score, topk)
47
+
48
+ triton_output = fused_moe(a, w1, w2, topk_weight, topk_ids, False)
49
+ torch_output = torch_moe(a, w1, w2, topk_weight, topk_ids)
50
+ assert torch.allclose(triton_output, torch_output, atol=1e-2, rtol=0)
vllm/model_executor/layers/fused_moe.py
new
+287
@@ -0,0 +1,287 @@
1
+"""Fused MoE kernel."""
2
+import torch
3
+import triton
4
+import triton.language as tl
5
+
6
+from vllm._C import ops
7
+
8
+
9
+@triton.jit
10
+def fused_moe_kernel(
11
+ # Pointers to matrices
12
+ a_ptr,
13
+ b_ptr,
14
+ c_ptr,
15
+ topk_weights_ptr,
16
+ sorted_token_ids_ptr,
17
+ expert_ids_ptr,
18
+ num_tokens_post_padded_ptr,
19
+ # Matrix dimensions
20
+ N,
21
+ K,
22
+ EM,
23
+ num_valid_tokens,
24
+ # The stride variables represent how much to increase the ptr by when moving by 1
25
+ # element in a particular dimension. E.g. `stride_am` is how much to increase `a_ptr`
26
+ # by to get the element one row down (A has M rows).
27
+ stride_am,
28
+ stride_ak,
29
+ stride_be,
30
+ stride_bk,
31
+ stride_bn,
32
+ stride_cm,
33
+ stride_cn,
34
+ # Meta-parameters
35
+ BLOCK_SIZE_M: tl.constexpr,
36
+ BLOCK_SIZE_N: tl.constexpr,
37
+ BLOCK_SIZE_K: tl.constexpr,
38
+ GROUP_SIZE_M: tl.constexpr,
39
+ MUL_ROUTED_WEIGHT: tl.constexpr,
40
+ top_k: tl.constexpr,
41
+ compute_type: tl.constexpr,
42
+):
43
+ """
44
+ Implements the fused computation for a Mixture of Experts (MOE) using token and expert matrices.
45
+
46
+ Key Parameters:
47
+ - A: The input tensor representing tokens with shape (*, K), where '*' can be any shape representing batches and K is the feature dimension of each token.
48
+ - B: The stacked MOE weight tensor with shape (E, N, K), where E is the number of experts, K is the input feature dimension, and N is the output feature dimension.
49
+ - C: The output cache tensor with shape (M, topk, N), where M is the total number of tokens post padding, topk is the number of times each token is repeated,
50
+ and N is the output feature dimension.
51
+ - sorted_token_ids: A tensor containing the sorted indices of tokens, repeated topk times and arranged by the expert index they are assigned to.
52
+ - expert_ids: A tensor containing the indices of the expert for each block. It determines which expert matrix from B should be used for each block in A.
53
+ This kernel performs the multiplication of a token by its corresponding expert matrix as determined by `expert_ids`. The sorting of `sorted_token_ids`
54
+ by expert index and padding ensures divisibility by BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix multiplication across different blocks processed by the same expert.
55
+ """
56
+ # -----------------------------------------------------------
57
+ # Map program ids `pid` to the block of C it should compute.
58
+ # This is done in a grouped ordering to promote L2 data reuse.
59
+ pid = tl.program_id(axis=0)
60
+ num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
61
+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
62
+ num_pid_in_group = GROUP_SIZE_M * num_pid_n
63
+ group_id = pid // num_pid_in_group
64
+ first_pid_m = group_id * GROUP_SIZE_M
65
+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
66
+ pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
67
+ pid_n = (pid % num_pid_in_group) // group_size_m
68
+
69
+ # ----------------------------------------------------------
70
+ # Create pointers for the first blocks of A and B.
71
+ # We will advance this pointer as we move in the K direction
72
+ # and accumulate
73
+ # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers
74
+ # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers
75
+ num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
76
+ if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
77
+ return
78
+ offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
79
+ offs_token = tl.load(sorted_token_ids_ptr + offs_token_id)
80
+ token_mask = offs_token < num_valid_tokens
81
+
82
+ offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
83
+ offs_k = tl.arange(0, BLOCK_SIZE_K)
84
+ a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am +
85
+ offs_k[None, :] * stride_ak)
86
+
87
+ off_experts = tl.load(expert_ids_ptr + pid_m)
88
+ b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
89
+ offs_bn[None, :] * stride_bn)
90
+
91
+ # -----------------------------------------------------------
92
+ # Iterate to compute a block of the C matrix.
93
+ # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
94
+ # of fp32 values for higher accuracy.
95
+ # `accumulator` will be converted back to fp16 after the loop.
96
+ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
97
+
98
+ for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
99
+ # Load the next block of A and B, generate a mask by checking the K dimension.
100
+ a = tl.load(a_ptrs,
101
+ mask=token_mask[:, None] &
102
+ (offs_k[None, :] < K - k * BLOCK_SIZE_K),
103
+ other=0.0)
104
+ b = tl.load(b_ptrs,
105
+ mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
106
+ other=0.0)
107
+ # We accumulate along the K dimension.
108
+ accumulator += tl.dot(a, b)
109
+ # Advance the ptrs to the next K block.
110
+ a_ptrs += BLOCK_SIZE_K * stride_ak
111
+ b_ptrs += BLOCK_SIZE_K * stride_bk
112
+
113
+ if MUL_ROUTED_WEIGHT:
114
+ moe_weight = tl.load(topk_weights_ptr + offs_token,
115
+ mask=token_mask,
116
+ other=0)
117
+ accumulator = accumulator * moe_weight[:, None]
118
+
119
+ accumulator = accumulator.to(compute_type)
120
+ # -----------------------------------------------------------
121
+ # Write back the block of the output
122
+ offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
123
+ c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[
124
+ None, :]
125
+ c_mask = token_mask[:, None] & (offs_cn[None, :] < N)
126
+ tl.store(c_ptrs, accumulator, mask=c_mask)
127
+
128
+
129
+def moe_align_block_size(
130
+ topk_ids: torch.Tensor, block_size: int,
131
+ num_experts: int) -> (torch.Tensor, torch.Tensor, torch.Tensor):
132
+ """
133
+ Aligns the token distribution across experts to be compatible with block size for matrix multiplication.
134
+
135
+ Parameters:
136
+ - topk_ids: A tensor of shape [total_tokens, top_k] representing the top-k expert indices for each token.
137
+ - block_size: The block size used in block matrix multiplication.
138
+ - num_experts: The total number of experts.
139
+
140
+ Returns:
141
+ - sorted_token_ids: A tensor containing the sorted token indices according to their allocated expert.
142
+ - expert_ids: A tensor indicating the assigned expert index for each block.
143
+ - num_tokens_post_padded: The total number of tokens after padding, ensuring divisibility by block_size.
144
+
145
+ This function pads the number of tokens that each expert needs to process so that it is divisible by block_size.
146
+ Padding ensures that during block matrix multiplication, the dimensions align correctly.
147
+
148
+ Example:
149
+ Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], block_size = 4, and num_experts = 4:
150
+ - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, with each expert needing to process 3 tokens.
151
+ - As block_size is 4, we pad 1 token for each expert.
152
+ - First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3].
153
+ - Then append padding tokens [12, 12, 12, 12] for each block.
154
+ - After sorting by expert index, we obtain token_ids [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12].
155
+ Tokens 12 are non-existent (padding) and are ignored in the subsequent matrix multiplication.
156
+ - The padding ensures that the total number of tokens is now divisible by block_size for proper block matrix operations.
157
+ """
158
+ sorted_ids = torch.empty(
159
+ (topk_ids.numel() + num_experts * (block_size - 1), ),
160
+ dtype=torch.int32,
161
+ device=topk_ids.device)
162
+ expert_ids = torch.empty((topk_ids.numel() + num_experts, ),
163
+ dtype=torch.int32,
164
+ device=topk_ids.device)
165
+ sorted_ids.fill_(topk_ids.numel())
166
+ num_tokens_post_pad = torch.empty((1),
167
+ dtype=torch.int32,
168
+ device=topk_ids.device)
169
+ ops.moe_align_block_size(topk_ids, num_experts, block_size, sorted_ids,
170
+ expert_ids, num_tokens_post_pad)
171
+ return sorted_ids, expert_ids, num_tokens_post_pad
172
+
173
+
174
+def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
175
+ topk_weights: torch.Tensor, topk_ids: torch.Tensor,
176
+ sorted_token_ids: torch.Tensor,
177
+ expert_ids: torch.Tensor,
178
+ num_tokens_post_padded: torch.Tensor,
179
+ mul_routed_weight: bool, top_k: int, config: dict):
180
+
181
+ assert topk_weights.stride(1) == 1
182
+ assert sorted_token_ids.stride(0) == 1
183
+
184
+ grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
185
+ 'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
186
+
187
+ fused_moe_kernel[grid](
188
+ A,
189
+ B,
190
+ C,
191
+ topk_weights,
192
+ sorted_token_ids,
193
+ expert_ids,
194
+ num_tokens_post_padded,
195
+ B.shape[1],
196
+ B.shape[2],
197
+ sorted_token_ids.shape[0],
198
+ topk_ids.numel(),
199
+ A.stride(0),
200
+ A.stride(1),
201
+ B.stride(0),
202
+ B.stride(2),
203
+ B.stride(1),
204
+ C.stride(1),
205
+ C.stride(2),
206
+ MUL_ROUTED_WEIGHT=mul_routed_weight,
207
+ top_k=top_k,
208
+ compute_type=tl.bfloat16 if A.dtype == torch.bfloat16 else tl.float16,
209
+ **config,
210
+ )
211
+
212
+
213
+def fused_moe(hidden_states: torch.Tensor,
214
+ w1: torch.Tensor,
215
+ w2: torch.Tensor,
216
+ topk_weights: torch.Tensor,
217
+ topk_ids: torch.Tensor,
218
+ inplace=False):
219
+ """
220
+ This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism.
221
+
222
+ Parameters:
223
+ - hidden_states (torch.Tensor): The input tensor to the MoE layer.
224
+ - w1 (torch.Tensor): The first set of expert weights.
225
+ - w2 (torch.Tensor): The second set of expert weights.
226
+ - topk_weights (torch.Tensor): The weights for the top-k selected experts.
227
+ - topk_ids (torch.Tensor): The indices of the top-k selected experts.
228
+ - inplace (bool): If True, perform the operation in-place. Defaults to False.
229
+
230
+ Returns:
231
+ - torch.Tensor: The output tensor after applying the MoE layer.
232
+ """
233
+ # Check constraints.
234
+ assert hidden_states.shape[1] == w1.shape[2], "Incompatible dimensions"
235
+ assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
236
+ assert w1.is_contiguous(), "Expert weights1 must be contiguous"
237
+ assert w2.is_contiguous(), "Expert weights2 must be contiguous"
238
+ assert hidden_states.dtype in [torch.float16, torch.bfloat16]
239
+ M, _ = hidden_states.shape
240
+ E, N, _ = w1.shape
241
+
242
+ config = {
243
+ 'BLOCK_SIZE_M': 64,
244
+ 'BLOCK_SIZE_N': 64,
245
+ 'BLOCK_SIZE_K': 32,
246
+ 'GROUP_SIZE_M': 8
247
+ }
248
+
249
+ if topk_ids.numel() <= w1.shape[0]:
250
+ config = {
251
+ 'BLOCK_SIZE_M': 16,
252
+ 'BLOCK_SIZE_N': 32,
253
+ 'BLOCK_SIZE_K': 64,
254
+ 'GROUP_SIZE_M': 1
255
+ }
256
+
257
+ intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N),
258
+ device=hidden_states.device,
259
+ dtype=hidden_states.dtype)
260
+ intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N // 2),
261
+ device=hidden_states.device,
262
+ dtype=hidden_states.dtype)
263
+ intermediate_cache3 = torch.empty((M, topk_ids.shape[1], w2.shape[1]),
264
+ device=hidden_states.device,
265
+ dtype=hidden_states.dtype)
266
+
267
+ sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
268
+ topk_ids, config['BLOCK_SIZE_M'], E)
269
+
270
+ invoke_fused_moe_kernel(hidden_states, w1, intermediate_cache1,
271
+ topk_weights, topk_ids, sorted_token_ids,
272
+ expert_ids, num_tokens_post_padded, False,
273
+ topk_ids.shape[1], config)
274
+
275
+ ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
276
+
277
+ invoke_fused_moe_kernel(intermediate_cache2, w2, intermediate_cache3,
278
+ topk_weights, topk_ids, sorted_token_ids,
279
+ expert_ids, num_tokens_post_padded, True, 1,
280
+ config)
281
+
282
+ if inplace:
283
+ return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
284
+ dim=1,
285
+ out=hidden_states)
286
+ return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
287
+ dim=1)
vllm/model_executor/models/__init__.py
+1
@@ -18,6 +18,7 @@ _MODELS = {
18
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
21
+ "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
22
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
23
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
24
"GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
vllm/model_executor/models/deepseek.py
new
+453
@@ -0,0 +1,453 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Deepseek model."""
24
+from typing import Any, Dict, List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+import torch.nn.functional as F
29
+from transformers import PretrainedConfig
30
+
31
+from vllm.model_executor.input_metadata import InputMetadata
32
+from vllm.model_executor.layers.activation import SiluAndMul
33
+from vllm.model_executor.layers.attention import PagedAttention
34
+from vllm.model_executor.layers.fused_moe import fused_moe
35
+from vllm.model_executor.layers.layernorm import RMSNorm
36
+from vllm.model_executor.layers.linear import (LinearMethodBase,
37
+ MergedColumnParallelLinear,
38
+ ReplicatedLinear,
39
+ QKVParallelLinear,
40
+ RowParallelLinear)
41
+from vllm.model_executor.layers.rotary_embedding import get_rope
42
+from vllm.model_executor.layers.sampler import Sampler
43
+from vllm.model_executor.layers.vocab_parallel_embedding import (
44
+ VocabParallelEmbedding, ParallelLMHead)
45
+from vllm.model_executor.parallel_utils.communication_op import (
46
+ tensor_model_parallel_all_reduce)
47
+from vllm.model_executor.parallel_utils.parallel_state import (
48
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49
+from vllm.model_executor.sampling_metadata import SamplingMetadata
50
+from vllm.model_executor.weight_utils import (default_weight_loader,
51
+ hf_model_weights_iterator)
52
+from vllm.sequence import SamplerOutput
53
+
54
+KVCache = Tuple[torch.Tensor, torch.Tensor]
55
+
56
+
57
+class DeepseekMLP(nn.Module):
58
+
59
+ def __init__(
60
+ self,
61
+ hidden_size: int,
62
+ intermediate_size: int,
63
+ hidden_act: str,
64
+ linear_method: Optional[LinearMethodBase] = None,
65
+ reduce_results: bool = True,
66
+ ) -> None:
67
+ super().__init__()
68
+ self.gate_up_proj = MergedColumnParallelLinear(
69
+ hidden_size, [intermediate_size] * 2,
70
+ bias=False,
71
+ linear_method=linear_method)
72
+ self.down_proj = RowParallelLinear(intermediate_size,
73
+ hidden_size,
74
+ bias=False,
75
+ linear_method=linear_method,
76
+ reduce_results=reduce_results)
77
+ if hidden_act != "silu":
78
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
79
+ "Only silu is supported for now.")
80
+ self.act_fn = SiluAndMul()
81
+
82
+ def forward(self, x):
83
+ gate_up, _ = self.gate_up_proj(x)
84
+ x = self.act_fn(gate_up)
85
+ x, _ = self.down_proj(x)
86
+ return x
87
+
88
+
89
+class DeepseekMoE(nn.Module):
90
+
91
+ def __init__(
92
+ self,
93
+ config: PretrainedConfig,
94
+ linear_method: Optional[LinearMethodBase] = None,
95
+ ):
96
+ super().__init__()
97
+ self.config = config
98
+ self.rank = get_tensor_model_parallel_rank()
99
+ self.tp_size = get_tensor_model_parallel_world_size()
100
+ self.n_routed_experts = config.n_routed_experts
101
+ self.top_k = config.num_experts_per_tok
102
+ if self.tp_size > self.n_routed_experts:
103
+ raise ValueError(
104
+ f"Tensor parallel size {self.tp_size} is greater than "
105
+ f"the number of experts {self.n_routed_experts}.")
106
+
107
+ self.experts = nn.ModuleList([
108
+ DeepseekMLP(hidden_size=config.hidden_size,
109
+ intermediate_size=config.moe_intermediate_size,
110
+ hidden_act=config.hidden_act,
111
+ linear_method=linear_method,
112
+ reduce_results=False)
113
+ for idx in range(self.n_routed_experts)
114
+ ])
115
+ self.pack_params()
116
+
117
+ self.gate = ReplicatedLinear(config.hidden_size,
118
+ self.n_routed_experts,
119
+ bias=False,
120
+ linear_method=None)
121
+
122
+ if config.n_shared_experts is not None:
123
+ intermediate_size = config.moe_intermediate_size * config.n_shared_experts
124
+ self.shared_experts = DeepseekMLP(
125
+ hidden_size=config.hidden_size,
126
+ intermediate_size=intermediate_size,
127
+ hidden_act=config.hidden_act,
128
+ linear_method=linear_method,
129
+ reduce_results=False,
130
+ )
131
+
132
+ def pack_params(self):
133
+ w1 = []
134
+ w2 = []
135
+ for expert in self.experts:
136
+ w1.append(expert.gate_up_proj.weight)
137
+ w2.append(expert.down_proj.weight)
138
+ self.w1 = torch._utils._flatten_dense_tensors(w1)
139
+ w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
140
+ for data, param in zip(w1s, w1):
141
+ param.data = data
142
+ self.w1 = self.w1.view(len(w1), *w1s[0].shape)
143
+
144
+ self.w2 = torch._utils._flatten_dense_tensors(w2)
145
+ w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
146
+ for data, param in zip(w2s, w2):
147
+ param.data = data
148
+
149
+ self.w2 = self.w2.view(len(w2), *w2s[0].shape)
150
+
151
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
152
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
153
+ hidden_states = hidden_states.view(-1, hidden_dim)
154
+ if self.config.n_shared_experts is not None:
155
+ shared_output = self.shared_experts(hidden_states)
156
+ # router_logits: (batch * sequence_length, n_experts)
157
+ router_logits, _ = self.gate(hidden_states)
158
+
159
+ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
160
+ routing_weights, selected_experts = torch.topk(routing_weights,
161
+ self.top_k,
162
+ dim=-1)
163
+
164
+ if self.config.norm_topk_prob:
165
+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
166
+
167
+ final_hidden_states = fused_moe(hidden_states,
168
+ self.w1,
169
+ self.w2,
170
+ routing_weights,
171
+ selected_experts,
172
+ inplace=True)
173
+
174
+ if self.config.n_shared_experts is not None:
175
+ final_hidden_states = final_hidden_states + shared_output
176
+ final_hidden_states = tensor_model_parallel_all_reduce(
177
+ final_hidden_states)
178
+
179
+ return final_hidden_states.view(batch_size, sequence_length,
180
+ hidden_dim)
181
+
182
+
183
+class DeepseekAttention(nn.Module):
184
+
185
+ def __init__(
186
+ self,
187
+ hidden_size: int,
188
+ num_heads: int,
189
+ num_kv_heads: int,
190
+ rope_theta: float = 10000,
191
+ rope_scaling: Optional[Dict[str, Any]] = None,
192
+ max_position_embeddings: int = 8192,
193
+ linear_method: Optional[LinearMethodBase] = None,
194
+ ) -> None:
195
+ super().__init__()
196
+ self.hidden_size = hidden_size
197
+ tp_size = get_tensor_model_parallel_world_size()
198
+ self.total_num_heads = num_heads
199
+ assert self.total_num_heads % tp_size == 0
200
+ self.num_heads = self.total_num_heads // tp_size
201
+ self.total_num_kv_heads = num_kv_heads
202
+ if self.total_num_kv_heads >= tp_size:
203
+ # Number of KV heads is greater than TP size, so we partition
204
+ # the KV heads across multiple tensor parallel GPUs.
205
+ assert self.total_num_kv_heads % tp_size == 0
206
+ else:
207
+ # Number of KV heads is less than TP size, so we replicate
208
+ # the KV heads across multiple tensor parallel GPUs.
209
+ assert tp_size % self.total_num_kv_heads == 0
210
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
211
+ self.head_dim = hidden_size // self.total_num_heads
212
+ self.q_size = self.num_heads * self.head_dim
213
+ self.kv_size = self.num_kv_heads * self.head_dim
214
+ self.scaling = self.head_dim**-0.5
215
+ self.rope_theta = rope_theta
216
+ self.max_position_embeddings = max_position_embeddings
217
+
218
+ self.qkv_proj = QKVParallelLinear(
219
+ hidden_size,
220
+ self.head_dim,
221
+ self.total_num_heads,
222
+ self.total_num_kv_heads,
223
+ bias=False,
224
+ linear_method=linear_method,
225
+ )
226
+
227
+ self.o_proj = RowParallelLinear(
228
+ self.total_num_heads * self.head_dim,
229
+ hidden_size,
230
+ bias=False,
231
+ linear_method=linear_method,
232
+ )
233
+
234
+ self.rotary_emb = get_rope(
235
+ self.head_dim,
236
+ rotary_dim=self.head_dim,
237
+ max_position=max_position_embeddings,
238
+ base=rope_theta,
239
+ rope_scaling=rope_scaling,
240
+ )
241
+ self.attn = PagedAttention(self.num_heads,
242
+ self.head_dim,
243
+ self.scaling,
244
+ num_kv_heads=self.num_kv_heads)
245
+
246
+ def forward(
247
+ self,
248
+ positions: torch.Tensor,
249
+ hidden_states: torch.Tensor,
250
+ kv_cache: KVCache,
251
+ input_metadata: InputMetadata,
252
+ ) -> torch.Tensor:
253
+ qkv, _ = self.qkv_proj(hidden_states)
254
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
255
+ q, k = self.rotary_emb(positions, q, k)
256
+ k_cache, v_cache = kv_cache
257
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
258
+ output, _ = self.o_proj(attn_output)
259
+ return output
260
+
261
+
262
+class DeepseekDecoderLayer(nn.Module):
263
+
264
+ def __init__(
265
+ self,
266
+ config: PretrainedConfig,
267
+ layer_idx: int,
268
+ linear_method: Optional[LinearMethodBase] = None,
269
+ ) -> None:
270
+ super().__init__()
271
+ self.hidden_size = config.hidden_size
272
+ rope_theta = getattr(config, "rope_theta", 10000)
273
+ rope_scaling = getattr(config, "rope_scaling", None)
274
+ max_position_embeddings = getattr(config, "max_position_embeddings",
275
+ 8192)
276
+ self.self_attn = DeepseekAttention(
277
+ hidden_size=self.hidden_size,
278
+ num_heads=config.num_attention_heads,
279
+ num_kv_heads=config.num_key_value_heads,
280
+ rope_theta=rope_theta,
281
+ rope_scaling=rope_scaling,
282
+ max_position_embeddings=max_position_embeddings,
283
+ linear_method=linear_method,
284
+ )
285
+ if (config.n_routed_experts is not None and \
286
+ layer_idx >= config.first_k_dense_replace and layer_idx % config.moe_layer_freq == 0):
287
+ self.mlp = DeepseekMoE(config=config, linear_method=linear_method)
288
+ else:
289
+ self.mlp = DeepseekMLP(
290
+ hidden_size=config.hidden_size,
291
+ intermediate_size=config.intermediate_size,
292
+ hidden_act=config.hidden_act,
293
+ linear_method=linear_method,
294
+ )
295
+ self.input_layernorm = RMSNorm(config.hidden_size,
296
+ eps=config.rms_norm_eps)
297
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
298
+ eps=config.rms_norm_eps)
299
+
300
+ def forward(
301
+ self,
302
+ positions: torch.Tensor,
303
+ hidden_states: torch.Tensor,
304
+ kv_cache: KVCache,
305
+ input_metadata: InputMetadata,
306
+ residual: Optional[torch.Tensor],
307
+ ) -> torch.Tensor:
308
+ # Self Attention
309
+ if residual is None:
310
+ residual = hidden_states
311
+ hidden_states = self.input_layernorm(hidden_states)
312
+ else:
313
+ hidden_states, residual = self.input_layernorm(
314
+ hidden_states, residual)
315
+ hidden_states = self.self_attn(
316
+ positions=positions,
317
+ hidden_states=hidden_states,
318
+ kv_cache=kv_cache,
319
+ input_metadata=input_metadata,
320
+ )
321
+
322
+ # Fully Connected
323
+ hidden_states, residual = self.post_attention_layernorm(
324
+ hidden_states, residual)
325
+ hidden_states = self.mlp(hidden_states)
326
+ return hidden_states, residual
327
+
328
+
329
+class DeepseekModel(nn.Module):
330
+
331
+ def __init__(
332
+ self,
333
+ config: PretrainedConfig,
334
+ linear_method: Optional[LinearMethodBase] = None,
335
+ ) -> None:
336
+ super().__init__()
337
+ self.padding_idx = config.pad_token_id
338
+ self.vocab_size = config.vocab_size
339
+
340
+ self.embed_tokens = VocabParallelEmbedding(
341
+ config.vocab_size,
342
+ config.hidden_size,
343
+ )
344
+ self.layers = nn.ModuleList([
345
+ DeepseekDecoderLayer(config,
346
+ layer_idx,
347
+ linear_method=linear_method)
348
+ for layer_idx in range(config.num_hidden_layers)
349
+ ])
350
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
351
+
352
+ def forward(
353
+ self,
354
+ input_ids: torch.Tensor,
355
+ positions: torch.Tensor,
356
+ kv_caches: List[KVCache],
357
+ input_metadata: InputMetadata,
358
+ ) -> torch.Tensor:
359
+ hidden_states = self.embed_tokens(input_ids)
360
+ residual = None
361
+ for i in range(len(self.layers)):
362
+ layer = self.layers[i]
363
+ hidden_states, residual = layer(positions, hidden_states,
364
+ kv_caches[i], input_metadata,
365
+ residual)
366
+ hidden_states, _ = self.norm(hidden_states, residual)
367
+ return hidden_states
368
+
369
+
370
+class DeepseekForCausalLM(nn.Module):
371
+
372
+ def __init__(
373
+ self,
374
+ config: PretrainedConfig,
375
+ linear_method: Optional[LinearMethodBase] = None,
376
+ ) -> None:
377
+ super().__init__()
378
+ self.config = config
379
+ self.linear_method = linear_method
380
+ self.model = DeepseekModel(config, linear_method)
381
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
382
+ self.sampler = Sampler(config.vocab_size)
383
+
384
+ def forward(
385
+ self,
386
+ input_ids: torch.Tensor,
387
+ positions: torch.Tensor,
388
+ kv_caches: List[KVCache],
389
+ input_metadata: InputMetadata,
390
+ ) -> torch.Tensor:
391
+ hidden_states = self.model(input_ids, positions, kv_caches,
392
+ input_metadata)
393
+ return hidden_states
394
+
395
+ def sample(
396
+ self,
397
+ hidden_states: Optional[torch.Tensor],
398
+ sampling_metadata: SamplingMetadata,
399
+ ) -> Optional[SamplerOutput]:
400
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
401
+ sampling_metadata)
402
+ return next_tokens
403
+
404
+ def load_weights(self,
405
+ model_name_or_path: str,
406
+ cache_dir: Optional[str] = None,
407
+ load_format: str = "auto",
408
+ revision: Optional[str] = None):
409
+ stacked_params_mapping = [
410
+ # (param_name, shard_name, shard_id)
411
+ ("qkv_proj", "q_proj", "q"),
412
+ ("qkv_proj", "k_proj", "k"),
413
+ ("qkv_proj", "v_proj", "v"),
414
+ ("gate_up_proj", "gate_proj", 0),
415
+ ("gate_up_proj", "up_proj", 1),
416
+ ]
417
+
418
+ params_dict = dict(self.named_parameters())
419
+ for name, loaded_weight in hf_model_weights_iterator(
420
+ model_name_or_path,
421
+ cache_dir,
422
+ load_format,
423
+ revision,
424
+ fall_back_to_pt=False):
425
+ if "rotary_emb.inv_freq" in name:
426
+ continue
427
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
428
+ if weight_name not in name:
429
+ continue
430
+ name = name.replace(weight_name, param_name)
431
+ # Skip loading extra bias for GPTQ models.
432
+ if name.endswith(".bias") and name not in params_dict:
433
+ continue
434
+ # Skip experts that are not assigned to this worker.
435
+ if (("mlp.experts." in name or "mlp.shared_experts." in name)
436
+ and name not in params_dict):
437
+ continue
438
+ param = params_dict[name]
439
+ weight_loader = param.weight_loader
440
+ weight_loader(param, loaded_weight, shard_id)
441
+ break
442
+ else:
443
+ # Skip loading extra bias for GPTQ models.
444
+ if name.endswith(".bias") and name not in params_dict:
445
+ continue
446
+ # Skip experts that are not assigned to this worker.
447
+ if (("mlp.experts." in name or "mlp.shared_experts." in name)
448
+ and name not in params_dict):
449
+ continue
450
+ param = params_dict[name]
451
+ weight_loader = getattr(param, "weight_loader",
452
+ default_weight_loader)
453
+ weight_loader(param, loaded_weight)