@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5d60def02c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

DeepseekMoE support with Fused MoE kernel (#2453)

Co-authored-by: roy <jasonailu87@gmail.com>

wangding zeng committed Jan 30, 2024 at 13:19 UTC 5d60def02cb5a43fa5864fcb123909b101df9ec5
9 files changed +924
csrc/dispatch_utils.h
+11
@@ -24,3 +24,14 @@
24 #define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
25 AT_DISPATCH_SWITCH( \
26 TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
27 +
28 +#define VLLM_DISPATCH_CASE_INTEGRAL_TYPES(...) \
29 + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \
30 + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \
31 + AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \
32 + AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \
33 + AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__)
34 +
35 +#define VLLM_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \
36 + AT_DISPATCH_SWITCH( \
37 + TYPE, NAME, VLLM_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__))
csrc/moe_align_block_size_kernels.cu new
+108
@@ -0,0 +1,108 @@
1 +#include <torch/extension.h>
2 +#include <ATen/cuda/CUDAContext.h>
3 +
4 +#include <ATen/ATen.h>
5 +#include <THC/THCAtomics.cuh>
6 +
7 +#include "cuda_compat.h"
8 +#include "dispatch_utils.h"
9 +
10 +const static size_t NUM_MAX_EXPERTS = 64;
11 +#define CEILDIV(x,y) (((x) + (y) - 1) / (y))
12 +
13 +namespace vllm {
14 +template <typename scalar_t>
15 +__global__ void moe_align_block_size_kernel(scalar_t *__restrict__ topk_ids,
16 + int32_t *sorted_token_ids,
17 + int32_t *expert_ids,
18 + int32_t *total_tokens_post_pad,
19 + int32_t num_experts,
20 + int32_t block_size,
21 + size_t numel) {
22 + const size_t tokens_per_thread = CEILDIV(numel, blockDim.x);
23 + const size_t start_idx = threadIdx.x * tokens_per_thread;
24 + __shared__ int32_t tokens_cnts[NUM_MAX_EXPERTS + 1][NUM_MAX_EXPERTS];
25 + __shared__ int32_t cumsum[NUM_MAX_EXPERTS + 1];
26 + for (int i = 0; i < num_experts; ++i) {
27 + tokens_cnts[threadIdx.x + 1][i] = 0;
28 + }
29 +
30 + /**
31 + * In the first step we compute token_cnts[thread_index + 1][expert_index],
32 + * which counts how many tokens in the token shard of thread_index are assigned
33 + * to expert expert_index.
34 + */
35 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
36 + ++tokens_cnts[threadIdx.x + 1][topk_ids[i]];
37 + }
38 +
39 + __syncthreads();
40 +
41 + // For each expert we accumulate the token counts from the different threads.
42 + tokens_cnts[0][threadIdx.x] = 0;
43 + for (int i = 1; i <= blockDim.x; ++i) {
44 + tokens_cnts[i][threadIdx.x] += tokens_cnts[i-1][threadIdx.x];
45 + }
46 +
47 + __syncthreads();
48 +
49 + // We accumulate the token counts of all experts in thread 0.
50 + if (threadIdx.x == 0) {
51 + cumsum[0] = 0;
52 + for (int i = 1; i <= num_experts; ++i) {
53 + cumsum[i] = cumsum[i-1] + CEILDIV(tokens_cnts[blockDim.x][i - 1], block_size) * block_size;
54 + }
55 + *total_tokens_post_pad = cumsum[num_experts];
56 + }
57 +
58 + __syncthreads();
59 +
60 + /**
61 + * For each expert, each thread processes the tokens of the corresponding blocks
62 + * and stores the corresponding expert_id for each block.
63 + */
64 + for (int i = cumsum[threadIdx.x];i < cumsum[threadIdx.x + 1];i += block_size) {
65 + expert_ids[i / block_size] = threadIdx.x;
66 + }
67 +
68 + /**
69 + * Each thread processes a token shard, calculating the index of each token after
70 + * sorting by expert number. Given the example topk_ids = [0,1,2,1,2,3,0,3,4] and
71 + * block_size = 4, then the output would be [0, 6, *, *, 1, 3, *, *, 2, 4, *, *, 5, 7, *, *, 8, *, *, *],
72 + * where * represents a padding value(preset in python).
73 + */
74 + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) {
75 + int32_t expert_id = topk_ids[i];
76 + /** The cumsum[expert_id] stores the starting index of the tokens that the
77 + * expert with expert_id needs to process, and tokens_cnts[threadIdx.x][expert_id]
78 + * stores the indices of the tokens processed by the expert with expert_id within
79 + * the current thread's token shard.
80 + */
81 + int32_t rank_post_pad = tokens_cnts[threadIdx.x][expert_id] + cumsum[expert_id];
82 + sorted_token_ids[rank_post_pad] = i;
83 + ++tokens_cnts[threadIdx.x][expert_id];
84 + }
85 +}
86 +}
87 +
88 +void moe_align_block_size(
89 + torch::Tensor topk_ids,
90 + int num_experts,
91 + int block_size,
92 + torch::Tensor sorted_token_ids,
93 + torch::Tensor experts_ids,
94 + torch::Tensor num_tokens_post_pad) {
95 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
96 + assert(num_experts <= NUM_MAX_EXPERTS);
97 + VLLM_DISPATCH_INTEGRAL_TYPES(
98 + topk_ids.scalar_type(), "moe_alig_block_size_kernel", [&] {
99 + vllm::moe_align_block_size_kernel<scalar_t><<<1, num_experts, 0, stream>>>(
100 + topk_ids.data_ptr<scalar_t>(),
101 + sorted_token_ids.data_ptr<int32_t>(),
102 + experts_ids.data_ptr<int32_t>(),
103 + num_tokens_post_pad.data_ptr<int32_t>(),
104 + num_experts,
105 + block_size,
106 + topk_ids.numel());
107 + });
108 +}
csrc/ops.h
+9
@@ -121,3 +121,12 @@ std::pair<std::vector<uint8_t>, std::vector<int64_t>> get_graph_buffer_ipc_meta(
121 void register_graph_buffers(fptr_t _fa, const std::vector<std::string> &handles,
122 const std::vector<std::vector<int64_t>> &offsets);
123 #endif
124 +
125 +void moe_align_block_size(
126 + torch::Tensor topk_ids,
127 + int num_experts,
128 + int block_size,
129 + torch::Tensor sorted_token_ids,
130 + torch::Tensor experts_ids,
131 + torch::Tensor num_tokens_post_pad
132 + );
csrc/pybind.cpp
+4
@@ -56,6 +56,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
56 ops.def("gptq_gemm", &gptq_gemm, "Quantized GEMM for GPTQ");
57 ops.def("gptq_shuffle", &gptq_shuffle, "Post processing for GPTQ");
58 ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
59 + ops.def(
60 + "moe_align_block_size",
61 + &moe_align_block_size,
62 + "Aligning the number of tokens to be processed by each expert such that it is divisible by the block size.");
63
64 // Cache ops
65 pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
setup.py
+1
@@ -309,6 +309,7 @@ vllm_extension_sources = [
309 "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
310 "csrc/quantization/gptq/q_gemm.cu",
311 "csrc/cuda_utils_kernels.cu",
312 + "csrc/moe_align_block_size_kernels.cu",
313 "csrc/pybind.cpp",
314 ]
315
tests/kernels/test_fused_moe.py new
+50
@@ -0,0 +1,50 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm.model_executor.layers.fused_moe import fused_moe
5 +from vllm.model_executor.layers.activation import SiluAndMul
6 +
7 +
8 +def torch_moe(a, w1, w2, topk_weight, topk_ids):
9 + B, D = a.shape
10 + a = a.view(B, -1, D).repeat(1, topk_ids.shape[1], 1).reshape(-1, D)
11 + out = torch.zeros(B * topk_ids.shape[1],
12 + w2.shape[1],
13 + dtype=a.dtype,
14 + device=a.device)
15 + topk_ids = topk_ids.view(-1)
16 + topk_weight = topk_weight.view(-1)
17 + for i in range(w1.shape[0]):
18 + mask = topk_ids == i
19 + if mask.sum():
20 + out[mask] = SiluAndMul()(
21 + a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
22 + return (out.view(B, -1, w2.shape[1]) *
23 + topk_weight.view(B, -1, 1)).sum(dim=1)
24 +
25 +
26 +@pytest.mark.parametrize("m", [512, 222, 33, 1])
27 +@pytest.mark.parametrize("n", [2048, 256, 1024])
28 +@pytest.mark.parametrize("k", [128, 511, 1024])
29 +@pytest.mark.parametrize("e", [8, 64])
30 +@pytest.mark.parametrize("topk", [2, 6])
31 +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
32 +def test_fused_moe(
33 + m: int,
34 + n: int,
35 + k: int,
36 + e: int,
37 + topk: int,
38 + dtype: torch.dtype,
39 +):
40 + a = torch.randn((m, k), device='cuda', dtype=dtype) / 10
41 + w1 = torch.randn((e, 2 * n, k), device='cuda', dtype=dtype) / 10
42 + w2 = torch.randn((e, k, n), device='cuda', dtype=dtype) / 10
43 +
44 + score = torch.randn((m, e), device='cuda', dtype=dtype)
45 + score = torch.softmax(score, dim=-1)
46 + topk_weight, topk_ids = torch.topk(score, topk)
47 +
48 + triton_output = fused_moe(a, w1, w2, topk_weight, topk_ids, False)
49 + torch_output = torch_moe(a, w1, w2, topk_weight, topk_ids)
50 + assert torch.allclose(triton_output, torch_output, atol=1e-2, rtol=0)
vllm/model_executor/layers/fused_moe.py new
+287
@@ -0,0 +1,287 @@
1 +"""Fused MoE kernel."""
2 +import torch
3 +import triton
4 +import triton.language as tl
5 +
6 +from vllm._C import ops
7 +
8 +
9 +@triton.jit
10 +def fused_moe_kernel(
11 + # Pointers to matrices
12 + a_ptr,
13 + b_ptr,
14 + c_ptr,
15 + topk_weights_ptr,
16 + sorted_token_ids_ptr,
17 + expert_ids_ptr,
18 + num_tokens_post_padded_ptr,
19 + # Matrix dimensions
20 + N,
21 + K,
22 + EM,
23 + num_valid_tokens,
24 + # The stride variables represent how much to increase the ptr by when moving by 1
25 + # element in a particular dimension. E.g. `stride_am` is how much to increase `a_ptr`
26 + # by to get the element one row down (A has M rows).
27 + stride_am,
28 + stride_ak,
29 + stride_be,
30 + stride_bk,
31 + stride_bn,
32 + stride_cm,
33 + stride_cn,
34 + # Meta-parameters
35 + BLOCK_SIZE_M: tl.constexpr,
36 + BLOCK_SIZE_N: tl.constexpr,
37 + BLOCK_SIZE_K: tl.constexpr,
38 + GROUP_SIZE_M: tl.constexpr,
39 + MUL_ROUTED_WEIGHT: tl.constexpr,
40 + top_k: tl.constexpr,
41 + compute_type: tl.constexpr,
42 +):
43 + """
44 + Implements the fused computation for a Mixture of Experts (MOE) using token and expert matrices.
45 +
46 + Key Parameters:
47 + - A: The input tensor representing tokens with shape (*, K), where '*' can be any shape representing batches and K is the feature dimension of each token.
48 + - B: The stacked MOE weight tensor with shape (E, N, K), where E is the number of experts, K is the input feature dimension, and N is the output feature dimension.
49 + - C: The output cache tensor with shape (M, topk, N), where M is the total number of tokens post padding, topk is the number of times each token is repeated,
50 + and N is the output feature dimension.
51 + - sorted_token_ids: A tensor containing the sorted indices of tokens, repeated topk times and arranged by the expert index they are assigned to.
52 + - expert_ids: A tensor containing the indices of the expert for each block. It determines which expert matrix from B should be used for each block in A.
53 + This kernel performs the multiplication of a token by its corresponding expert matrix as determined by `expert_ids`. The sorting of `sorted_token_ids`
54 + by expert index and padding ensures divisibility by BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix multiplication across different blocks processed by the same expert.
55 + """
56 + # -----------------------------------------------------------
57 + # Map program ids `pid` to the block of C it should compute.
58 + # This is done in a grouped ordering to promote L2 data reuse.
59 + pid = tl.program_id(axis=0)
60 + num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M)
61 + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
62 + num_pid_in_group = GROUP_SIZE_M * num_pid_n
63 + group_id = pid // num_pid_in_group
64 + first_pid_m = group_id * GROUP_SIZE_M
65 + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
66 + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
67 + pid_n = (pid % num_pid_in_group) // group_size_m
68 +
69 + # ----------------------------------------------------------
70 + # Create pointers for the first blocks of A and B.
71 + # We will advance this pointer as we move in the K direction
72 + # and accumulate
73 + # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers
74 + # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers
75 + num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr)
76 + if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
77 + return
78 + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
79 + offs_token = tl.load(sorted_token_ids_ptr + offs_token_id)
80 + token_mask = offs_token < num_valid_tokens
81 +
82 + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
83 + offs_k = tl.arange(0, BLOCK_SIZE_K)
84 + a_ptrs = a_ptr + (offs_token[:, None] // top_k * stride_am +
85 + offs_k[None, :] * stride_ak)
86 +
87 + off_experts = tl.load(expert_ids_ptr + pid_m)
88 + b_ptrs = b_ptr + off_experts * stride_be + (offs_k[:, None] * stride_bk +
89 + offs_bn[None, :] * stride_bn)
90 +
91 + # -----------------------------------------------------------
92 + # Iterate to compute a block of the C matrix.
93 + # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
94 + # of fp32 values for higher accuracy.
95 + # `accumulator` will be converted back to fp16 after the loop.
96 + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
97 +
98 + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
99 + # Load the next block of A and B, generate a mask by checking the K dimension.
100 + a = tl.load(a_ptrs,
101 + mask=token_mask[:, None] &
102 + (offs_k[None, :] < K - k * BLOCK_SIZE_K),
103 + other=0.0)
104 + b = tl.load(b_ptrs,
105 + mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
106 + other=0.0)
107 + # We accumulate along the K dimension.
108 + accumulator += tl.dot(a, b)
109 + # Advance the ptrs to the next K block.
110 + a_ptrs += BLOCK_SIZE_K * stride_ak
111 + b_ptrs += BLOCK_SIZE_K * stride_bk
112 +
113 + if MUL_ROUTED_WEIGHT:
114 + moe_weight = tl.load(topk_weights_ptr + offs_token,
115 + mask=token_mask,
116 + other=0)
117 + accumulator = accumulator * moe_weight[:, None]
118 +
119 + accumulator = accumulator.to(compute_type)
120 + # -----------------------------------------------------------
121 + # Write back the block of the output
122 + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
123 + c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[
124 + None, :]
125 + c_mask = token_mask[:, None] & (offs_cn[None, :] < N)
126 + tl.store(c_ptrs, accumulator, mask=c_mask)
127 +
128 +
129 +def moe_align_block_size(
130 + topk_ids: torch.Tensor, block_size: int,
131 + num_experts: int) -> (torch.Tensor, torch.Tensor, torch.Tensor):
132 + """
133 + Aligns the token distribution across experts to be compatible with block size for matrix multiplication.
134 +
135 + Parameters:
136 + - topk_ids: A tensor of shape [total_tokens, top_k] representing the top-k expert indices for each token.
137 + - block_size: The block size used in block matrix multiplication.
138 + - num_experts: The total number of experts.
139 +
140 + Returns:
141 + - sorted_token_ids: A tensor containing the sorted token indices according to their allocated expert.
142 + - expert_ids: A tensor indicating the assigned expert index for each block.
143 + - num_tokens_post_padded: The total number of tokens after padding, ensuring divisibility by block_size.
144 +
145 + This function pads the number of tokens that each expert needs to process so that it is divisible by block_size.
146 + Padding ensures that during block matrix multiplication, the dimensions align correctly.
147 +
148 + Example:
149 + Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], block_size = 4, and num_experts = 4:
150 + - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, with each expert needing to process 3 tokens.
151 + - As block_size is 4, we pad 1 token for each expert.
152 + - First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3].
153 + - Then append padding tokens [12, 12, 12, 12] for each block.
154 + - After sorting by expert index, we obtain token_ids [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12].
155 + Tokens 12 are non-existent (padding) and are ignored in the subsequent matrix multiplication.
156 + - The padding ensures that the total number of tokens is now divisible by block_size for proper block matrix operations.
157 + """
158 + sorted_ids = torch.empty(
159 + (topk_ids.numel() + num_experts * (block_size - 1), ),
160 + dtype=torch.int32,
161 + device=topk_ids.device)
162 + expert_ids = torch.empty((topk_ids.numel() + num_experts, ),
163 + dtype=torch.int32,
164 + device=topk_ids.device)
165 + sorted_ids.fill_(topk_ids.numel())
166 + num_tokens_post_pad = torch.empty((1),
167 + dtype=torch.int32,
168 + device=topk_ids.device)
169 + ops.moe_align_block_size(topk_ids, num_experts, block_size, sorted_ids,
170 + expert_ids, num_tokens_post_pad)
171 + return sorted_ids, expert_ids, num_tokens_post_pad
172 +
173 +
174 +def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
175 + topk_weights: torch.Tensor, topk_ids: torch.Tensor,
176 + sorted_token_ids: torch.Tensor,
177 + expert_ids: torch.Tensor,
178 + num_tokens_post_padded: torch.Tensor,
179 + mul_routed_weight: bool, top_k: int, config: dict):
180 +
181 + assert topk_weights.stride(1) == 1
182 + assert sorted_token_ids.stride(0) == 1
183 +
184 + grid = lambda META: (triton.cdiv(sorted_token_ids.shape[0], META[
185 + 'BLOCK_SIZE_M']) * triton.cdiv(B.shape[1], META['BLOCK_SIZE_N']), )
186 +
187 + fused_moe_kernel[grid](
188 + A,
189 + B,
190 + C,
191 + topk_weights,
192 + sorted_token_ids,
193 + expert_ids,
194 + num_tokens_post_padded,
195 + B.shape[1],
196 + B.shape[2],
197 + sorted_token_ids.shape[0],
198 + topk_ids.numel(),
199 + A.stride(0),
200 + A.stride(1),
201 + B.stride(0),
202 + B.stride(2),
203 + B.stride(1),
204 + C.stride(1),
205 + C.stride(2),
206 + MUL_ROUTED_WEIGHT=mul_routed_weight,
207 + top_k=top_k,
208 + compute_type=tl.bfloat16 if A.dtype == torch.bfloat16 else tl.float16,
209 + **config,
210 + )
211 +
212 +
213 +def fused_moe(hidden_states: torch.Tensor,
214 + w1: torch.Tensor,
215 + w2: torch.Tensor,
216 + topk_weights: torch.Tensor,
217 + topk_ids: torch.Tensor,
218 + inplace=False):
219 + """
220 + This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism.
221 +
222 + Parameters:
223 + - hidden_states (torch.Tensor): The input tensor to the MoE layer.
224 + - w1 (torch.Tensor): The first set of expert weights.
225 + - w2 (torch.Tensor): The second set of expert weights.
226 + - topk_weights (torch.Tensor): The weights for the top-k selected experts.
227 + - topk_ids (torch.Tensor): The indices of the top-k selected experts.
228 + - inplace (bool): If True, perform the operation in-place. Defaults to False.
229 +
230 + Returns:
231 + - torch.Tensor: The output tensor after applying the MoE layer.
232 + """
233 + # Check constraints.
234 + assert hidden_states.shape[1] == w1.shape[2], "Incompatible dimensions"
235 + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
236 + assert w1.is_contiguous(), "Expert weights1 must be contiguous"
237 + assert w2.is_contiguous(), "Expert weights2 must be contiguous"
238 + assert hidden_states.dtype in [torch.float16, torch.bfloat16]
239 + M, _ = hidden_states.shape
240 + E, N, _ = w1.shape
241 +
242 + config = {
243 + 'BLOCK_SIZE_M': 64,
244 + 'BLOCK_SIZE_N': 64,
245 + 'BLOCK_SIZE_K': 32,
246 + 'GROUP_SIZE_M': 8
247 + }
248 +
249 + if topk_ids.numel() <= w1.shape[0]:
250 + config = {
251 + 'BLOCK_SIZE_M': 16,
252 + 'BLOCK_SIZE_N': 32,
253 + 'BLOCK_SIZE_K': 64,
254 + 'GROUP_SIZE_M': 1
255 + }
256 +
257 + intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N),
258 + device=hidden_states.device,
259 + dtype=hidden_states.dtype)
260 + intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N // 2),
261 + device=hidden_states.device,
262 + dtype=hidden_states.dtype)
263 + intermediate_cache3 = torch.empty((M, topk_ids.shape[1], w2.shape[1]),
264 + device=hidden_states.device,
265 + dtype=hidden_states.dtype)
266 +
267 + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
268 + topk_ids, config['BLOCK_SIZE_M'], E)
269 +
270 + invoke_fused_moe_kernel(hidden_states, w1, intermediate_cache1,
271 + topk_weights, topk_ids, sorted_token_ids,
272 + expert_ids, num_tokens_post_padded, False,
273 + topk_ids.shape[1], config)
274 +
275 + ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
276 +
277 + invoke_fused_moe_kernel(intermediate_cache2, w2, intermediate_cache3,
278 + topk_weights, topk_ids, sorted_token_ids,
279 + expert_ids, num_tokens_post_padded, True, 1,
280 + config)
281 +
282 + if inplace:
283 + return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
284 + dim=1,
285 + out=hidden_states)
286 + return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
287 + dim=1)
vllm/model_executor/models/__init__.py
+1
@@ -18,6 +18,7 @@ _MODELS = {
18 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
21 + "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
22 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
23 "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
24 "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
vllm/model_executor/models/deepseek.py new
+453
@@ -0,0 +1,453 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Deepseek model."""
24 +from typing import Any, Dict, List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +import torch.nn.functional as F
29 +from transformers import PretrainedConfig
30 +
31 +from vllm.model_executor.input_metadata import InputMetadata
32 +from vllm.model_executor.layers.activation import SiluAndMul
33 +from vllm.model_executor.layers.attention import PagedAttention
34 +from vllm.model_executor.layers.fused_moe import fused_moe
35 +from vllm.model_executor.layers.layernorm import RMSNorm
36 +from vllm.model_executor.layers.linear import (LinearMethodBase,
37 + MergedColumnParallelLinear,
38 + ReplicatedLinear,
39 + QKVParallelLinear,
40 + RowParallelLinear)
41 +from vllm.model_executor.layers.rotary_embedding import get_rope
42 +from vllm.model_executor.layers.sampler import Sampler
43 +from vllm.model_executor.layers.vocab_parallel_embedding import (
44 + VocabParallelEmbedding, ParallelLMHead)
45 +from vllm.model_executor.parallel_utils.communication_op import (
46 + tensor_model_parallel_all_reduce)
47 +from vllm.model_executor.parallel_utils.parallel_state import (
48 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49 +from vllm.model_executor.sampling_metadata import SamplingMetadata
50 +from vllm.model_executor.weight_utils import (default_weight_loader,
51 + hf_model_weights_iterator)
52 +from vllm.sequence import SamplerOutput
53 +
54 +KVCache = Tuple[torch.Tensor, torch.Tensor]
55 +
56 +
57 +class DeepseekMLP(nn.Module):
58 +
59 + def __init__(
60 + self,
61 + hidden_size: int,
62 + intermediate_size: int,
63 + hidden_act: str,
64 + linear_method: Optional[LinearMethodBase] = None,
65 + reduce_results: bool = True,
66 + ) -> None:
67 + super().__init__()
68 + self.gate_up_proj = MergedColumnParallelLinear(
69 + hidden_size, [intermediate_size] * 2,
70 + bias=False,
71 + linear_method=linear_method)
72 + self.down_proj = RowParallelLinear(intermediate_size,
73 + hidden_size,
74 + bias=False,
75 + linear_method=linear_method,
76 + reduce_results=reduce_results)
77 + if hidden_act != "silu":
78 + raise ValueError(f"Unsupported activation: {hidden_act}. "
79 + "Only silu is supported for now.")
80 + self.act_fn = SiluAndMul()
81 +
82 + def forward(self, x):
83 + gate_up, _ = self.gate_up_proj(x)
84 + x = self.act_fn(gate_up)
85 + x, _ = self.down_proj(x)
86 + return x
87 +
88 +
89 +class DeepseekMoE(nn.Module):
90 +
91 + def __init__(
92 + self,
93 + config: PretrainedConfig,
94 + linear_method: Optional[LinearMethodBase] = None,
95 + ):
96 + super().__init__()
97 + self.config = config
98 + self.rank = get_tensor_model_parallel_rank()
99 + self.tp_size = get_tensor_model_parallel_world_size()
100 + self.n_routed_experts = config.n_routed_experts
101 + self.top_k = config.num_experts_per_tok
102 + if self.tp_size > self.n_routed_experts:
103 + raise ValueError(
104 + f"Tensor parallel size {self.tp_size} is greater than "
105 + f"the number of experts {self.n_routed_experts}.")
106 +
107 + self.experts = nn.ModuleList([
108 + DeepseekMLP(hidden_size=config.hidden_size,
109 + intermediate_size=config.moe_intermediate_size,
110 + hidden_act=config.hidden_act,
111 + linear_method=linear_method,
112 + reduce_results=False)
113 + for idx in range(self.n_routed_experts)
114 + ])
115 + self.pack_params()
116 +
117 + self.gate = ReplicatedLinear(config.hidden_size,
118 + self.n_routed_experts,
119 + bias=False,
120 + linear_method=None)
121 +
122 + if config.n_shared_experts is not None:
123 + intermediate_size = config.moe_intermediate_size * config.n_shared_experts
124 + self.shared_experts = DeepseekMLP(
125 + hidden_size=config.hidden_size,
126 + intermediate_size=intermediate_size,
127 + hidden_act=config.hidden_act,
128 + linear_method=linear_method,
129 + reduce_results=False,
130 + )
131 +
132 + def pack_params(self):
133 + w1 = []
134 + w2 = []
135 + for expert in self.experts:
136 + w1.append(expert.gate_up_proj.weight)
137 + w2.append(expert.down_proj.weight)
138 + self.w1 = torch._utils._flatten_dense_tensors(w1)
139 + w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
140 + for data, param in zip(w1s, w1):
141 + param.data = data
142 + self.w1 = self.w1.view(len(w1), *w1s[0].shape)
143 +
144 + self.w2 = torch._utils._flatten_dense_tensors(w2)
145 + w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
146 + for data, param in zip(w2s, w2):
147 + param.data = data
148 +
149 + self.w2 = self.w2.view(len(w2), *w2s[0].shape)
150 +
151 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
152 + batch_size, sequence_length, hidden_dim = hidden_states.shape
153 + hidden_states = hidden_states.view(-1, hidden_dim)
154 + if self.config.n_shared_experts is not None:
155 + shared_output = self.shared_experts(hidden_states)
156 + # router_logits: (batch * sequence_length, n_experts)
157 + router_logits, _ = self.gate(hidden_states)
158 +
159 + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
160 + routing_weights, selected_experts = torch.topk(routing_weights,
161 + self.top_k,
162 + dim=-1)
163 +
164 + if self.config.norm_topk_prob:
165 + routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
166 +
167 + final_hidden_states = fused_moe(hidden_states,
168 + self.w1,
169 + self.w2,
170 + routing_weights,
171 + selected_experts,
172 + inplace=True)
173 +
174 + if self.config.n_shared_experts is not None:
175 + final_hidden_states = final_hidden_states + shared_output
176 + final_hidden_states = tensor_model_parallel_all_reduce(
177 + final_hidden_states)
178 +
179 + return final_hidden_states.view(batch_size, sequence_length,
180 + hidden_dim)
181 +
182 +
183 +class DeepseekAttention(nn.Module):
184 +
185 + def __init__(
186 + self,
187 + hidden_size: int,
188 + num_heads: int,
189 + num_kv_heads: int,
190 + rope_theta: float = 10000,
191 + rope_scaling: Optional[Dict[str, Any]] = None,
192 + max_position_embeddings: int = 8192,
193 + linear_method: Optional[LinearMethodBase] = None,
194 + ) -> None:
195 + super().__init__()
196 + self.hidden_size = hidden_size
197 + tp_size = get_tensor_model_parallel_world_size()
198 + self.total_num_heads = num_heads
199 + assert self.total_num_heads % tp_size == 0
200 + self.num_heads = self.total_num_heads // tp_size
201 + self.total_num_kv_heads = num_kv_heads
202 + if self.total_num_kv_heads >= tp_size:
203 + # Number of KV heads is greater than TP size, so we partition
204 + # the KV heads across multiple tensor parallel GPUs.
205 + assert self.total_num_kv_heads % tp_size == 0
206 + else:
207 + # Number of KV heads is less than TP size, so we replicate
208 + # the KV heads across multiple tensor parallel GPUs.
209 + assert tp_size % self.total_num_kv_heads == 0
210 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
211 + self.head_dim = hidden_size // self.total_num_heads
212 + self.q_size = self.num_heads * self.head_dim
213 + self.kv_size = self.num_kv_heads * self.head_dim
214 + self.scaling = self.head_dim**-0.5
215 + self.rope_theta = rope_theta
216 + self.max_position_embeddings = max_position_embeddings
217 +
218 + self.qkv_proj = QKVParallelLinear(
219 + hidden_size,
220 + self.head_dim,
221 + self.total_num_heads,
222 + self.total_num_kv_heads,
223 + bias=False,
224 + linear_method=linear_method,
225 + )
226 +
227 + self.o_proj = RowParallelLinear(
228 + self.total_num_heads * self.head_dim,
229 + hidden_size,
230 + bias=False,
231 + linear_method=linear_method,
232 + )
233 +
234 + self.rotary_emb = get_rope(
235 + self.head_dim,
236 + rotary_dim=self.head_dim,
237 + max_position=max_position_embeddings,
238 + base=rope_theta,
239 + rope_scaling=rope_scaling,
240 + )
241 + self.attn = PagedAttention(self.num_heads,
242 + self.head_dim,
243 + self.scaling,
244 + num_kv_heads=self.num_kv_heads)
245 +
246 + def forward(
247 + self,
248 + positions: torch.Tensor,
249 + hidden_states: torch.Tensor,
250 + kv_cache: KVCache,
251 + input_metadata: InputMetadata,
252 + ) -> torch.Tensor:
253 + qkv, _ = self.qkv_proj(hidden_states)
254 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
255 + q, k = self.rotary_emb(positions, q, k)
256 + k_cache, v_cache = kv_cache
257 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
258 + output, _ = self.o_proj(attn_output)
259 + return output
260 +
261 +
262 +class DeepseekDecoderLayer(nn.Module):
263 +
264 + def __init__(
265 + self,
266 + config: PretrainedConfig,
267 + layer_idx: int,
268 + linear_method: Optional[LinearMethodBase] = None,
269 + ) -> None:
270 + super().__init__()
271 + self.hidden_size = config.hidden_size
272 + rope_theta = getattr(config, "rope_theta", 10000)
273 + rope_scaling = getattr(config, "rope_scaling", None)
274 + max_position_embeddings = getattr(config, "max_position_embeddings",
275 + 8192)
276 + self.self_attn = DeepseekAttention(
277 + hidden_size=self.hidden_size,
278 + num_heads=config.num_attention_heads,
279 + num_kv_heads=config.num_key_value_heads,
280 + rope_theta=rope_theta,
281 + rope_scaling=rope_scaling,
282 + max_position_embeddings=max_position_embeddings,
283 + linear_method=linear_method,
284 + )
285 + if (config.n_routed_experts is not None and \
286 + layer_idx >= config.first_k_dense_replace and layer_idx % config.moe_layer_freq == 0):
287 + self.mlp = DeepseekMoE(config=config, linear_method=linear_method)
288 + else:
289 + self.mlp = DeepseekMLP(
290 + hidden_size=config.hidden_size,
291 + intermediate_size=config.intermediate_size,
292 + hidden_act=config.hidden_act,
293 + linear_method=linear_method,
294 + )
295 + self.input_layernorm = RMSNorm(config.hidden_size,
296 + eps=config.rms_norm_eps)
297 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
298 + eps=config.rms_norm_eps)
299 +
300 + def forward(
301 + self,
302 + positions: torch.Tensor,
303 + hidden_states: torch.Tensor,
304 + kv_cache: KVCache,
305 + input_metadata: InputMetadata,
306 + residual: Optional[torch.Tensor],
307 + ) -> torch.Tensor:
308 + # Self Attention
309 + if residual is None:
310 + residual = hidden_states
311 + hidden_states = self.input_layernorm(hidden_states)
312 + else:
313 + hidden_states, residual = self.input_layernorm(
314 + hidden_states, residual)
315 + hidden_states = self.self_attn(
316 + positions=positions,
317 + hidden_states=hidden_states,
318 + kv_cache=kv_cache,
319 + input_metadata=input_metadata,
320 + )
321 +
322 + # Fully Connected
323 + hidden_states, residual = self.post_attention_layernorm(
324 + hidden_states, residual)
325 + hidden_states = self.mlp(hidden_states)
326 + return hidden_states, residual
327 +
328 +
329 +class DeepseekModel(nn.Module):
330 +
331 + def __init__(
332 + self,
333 + config: PretrainedConfig,
334 + linear_method: Optional[LinearMethodBase] = None,
335 + ) -> None:
336 + super().__init__()
337 + self.padding_idx = config.pad_token_id
338 + self.vocab_size = config.vocab_size
339 +
340 + self.embed_tokens = VocabParallelEmbedding(
341 + config.vocab_size,
342 + config.hidden_size,
343 + )
344 + self.layers = nn.ModuleList([
345 + DeepseekDecoderLayer(config,
346 + layer_idx,
347 + linear_method=linear_method)
348 + for layer_idx in range(config.num_hidden_layers)
349 + ])
350 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
351 +
352 + def forward(
353 + self,
354 + input_ids: torch.Tensor,
355 + positions: torch.Tensor,
356 + kv_caches: List[KVCache],
357 + input_metadata: InputMetadata,
358 + ) -> torch.Tensor:
359 + hidden_states = self.embed_tokens(input_ids)
360 + residual = None
361 + for i in range(len(self.layers)):
362 + layer = self.layers[i]
363 + hidden_states, residual = layer(positions, hidden_states,
364 + kv_caches[i], input_metadata,
365 + residual)
366 + hidden_states, _ = self.norm(hidden_states, residual)
367 + return hidden_states
368 +
369 +
370 +class DeepseekForCausalLM(nn.Module):
371 +
372 + def __init__(
373 + self,
374 + config: PretrainedConfig,
375 + linear_method: Optional[LinearMethodBase] = None,
376 + ) -> None:
377 + super().__init__()
378 + self.config = config
379 + self.linear_method = linear_method
380 + self.model = DeepseekModel(config, linear_method)
381 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
382 + self.sampler = Sampler(config.vocab_size)
383 +
384 + def forward(
385 + self,
386 + input_ids: torch.Tensor,
387 + positions: torch.Tensor,
388 + kv_caches: List[KVCache],
389 + input_metadata: InputMetadata,
390 + ) -> torch.Tensor:
391 + hidden_states = self.model(input_ids, positions, kv_caches,
392 + input_metadata)
393 + return hidden_states
394 +
395 + def sample(
396 + self,
397 + hidden_states: Optional[torch.Tensor],
398 + sampling_metadata: SamplingMetadata,
399 + ) -> Optional[SamplerOutput]:
400 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
401 + sampling_metadata)
402 + return next_tokens
403 +
404 + def load_weights(self,
405 + model_name_or_path: str,
406 + cache_dir: Optional[str] = None,
407 + load_format: str = "auto",
408 + revision: Optional[str] = None):
409 + stacked_params_mapping = [
410 + # (param_name, shard_name, shard_id)
411 + ("qkv_proj", "q_proj", "q"),
412 + ("qkv_proj", "k_proj", "k"),
413 + ("qkv_proj", "v_proj", "v"),
414 + ("gate_up_proj", "gate_proj", 0),
415 + ("gate_up_proj", "up_proj", 1),
416 + ]
417 +
418 + params_dict = dict(self.named_parameters())
419 + for name, loaded_weight in hf_model_weights_iterator(
420 + model_name_or_path,
421 + cache_dir,
422 + load_format,
423 + revision,
424 + fall_back_to_pt=False):
425 + if "rotary_emb.inv_freq" in name:
426 + continue
427 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
428 + if weight_name not in name:
429 + continue
430 + name = name.replace(weight_name, param_name)
431 + # Skip loading extra bias for GPTQ models.
432 + if name.endswith(".bias") and name not in params_dict:
433 + continue
434 + # Skip experts that are not assigned to this worker.
435 + if (("mlp.experts." in name or "mlp.shared_experts." in name)
436 + and name not in params_dict):
437 + continue
438 + param = params_dict[name]
439 + weight_loader = param.weight_loader
440 + weight_loader(param, loaded_weight, shard_id)
441 + break
442 + else:
443 + # Skip loading extra bias for GPTQ models.
444 + if name.endswith(".bias") and name not in params_dict:
445 + continue
446 + # Skip experts that are not assigned to this worker.
447 + if (("mlp.experts." in name or "mlp.shared_experts." in name)
448 + and name not in params_dict):
449 + continue
450 + param = params_dict[name]
451 + weight_loader = getattr(param, "weight_loader",
452 + default_weight_loader)
453 + weight_loader(param, loaded_weight)