@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0d4e14557
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add fused top-K softmax kernel for MoE (#2769)

Woosuk Kwon committed Feb 5, 2024 at 17:38 UTC f0d4e145575bf6fb96c141d776ce92c9bfc79c49
9 files changed +591 -50
csrc/moe/moe_ops.cpp new
+7
@@ -0,0 +1,7 @@
1 +#include "moe_ops.h"
2 +
3 +#include <torch/extension.h>
4 +
5 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
6 + m.def("topk_softmax", &topk_softmax, "Apply topk softmax to the gating outputs.");
7 +}
csrc/moe/moe_ops.h new
+9
@@ -0,0 +1,9 @@
1 +#pragma once
2 +
3 +#include <torch/extension.h>
4 +
5 +void topk_softmax(
6 + torch::Tensor& topk_weights,
7 + torch::Tensor& topk_indices,
8 + torch::Tensor& token_expert_indices,
9 + torch::Tensor& gating_output);
csrc/moe/topk_softmax_kernels.cu new
+499
@@ -0,0 +1,499 @@
1 +/*
2 + * Adapted from https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu
3 + * Copyright (c) 2024, The vLLM team.
4 + * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 + * SPDX-License-Identifier: Apache-2.0
6 + *
7 + * Licensed under the Apache License, Version 2.0 (the "License");
8 + * you may not use this file except in compliance with the License.
9 + * You may obtain a copy of the License at
10 + *
11 + * http://www.apache.org/licenses/LICENSE-2.0
12 + *
13 + * Unless required by applicable law or agreed to in writing, software
14 + * distributed under the License is distributed on an "AS IS" BASIS,
15 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 + * See the License for the specific language governing permissions and
17 + * limitations under the License.
18 + */
19 +#include <torch/extension.h>
20 +#include <ATen/cuda/CUDAContext.h>
21 +#include <c10/cuda/CUDAGuard.h>
22 +
23 +#include <cub/cub.cuh>
24 +#include <cub/util_type.cuh>
25 +
26 +namespace vllm {
27 +namespace moe {
28 +
29 +static constexpr int WARP_SIZE = 32;
30 +
31 +/// Aligned array type
32 +template <
33 + typename T,
34 + /// Number of elements in the array
35 + int N,
36 + /// Alignment requirement in bytes
37 + int Alignment = sizeof(T) * N
38 +>
39 +class alignas(Alignment) AlignedArray {
40 + float data[N];
41 +};
42 +
43 +// ====================== Softmax things ===============================
44 +// We have our own implementation of softmax here so we can support transposing the output
45 +// in the softmax kernel when we extend this module to support expert-choice routing.
46 +template <int TPB>
47 +__launch_bounds__(TPB) __global__
48 + void moeSoftmax(const float* input, const bool* finished, float* output, const int num_cols)
49 +{
50 + using BlockReduce = cub::BlockReduce<float, TPB>;
51 + __shared__ typename BlockReduce::TempStorage tmpStorage;
52 +
53 + __shared__ float normalizing_factor;
54 + __shared__ float float_max;
55 +
56 + const int thread_row_offset = blockIdx.x * num_cols;
57 +
58 + cub::Sum sum;
59 + float threadData(-FLT_MAX);
60 +
61 + // Don't touch finished rows.
62 + if ((finished != nullptr) && finished[blockIdx.x])
63 + {
64 + return;
65 + }
66 +
67 + for (int ii = threadIdx.x; ii < num_cols; ii += TPB)
68 + {
69 + const int idx = thread_row_offset + ii;
70 + threadData = max(static_cast<float>(input[idx]), threadData);
71 + }
72 +
73 + const float maxElem = BlockReduce(tmpStorage).Reduce(threadData, cub::Max());
74 + if (threadIdx.x == 0)
75 + {
76 + float_max = maxElem;
77 + }
78 + __syncthreads();
79 +
80 + threadData = 0;
81 +
82 + for (int ii = threadIdx.x; ii < num_cols; ii += TPB)
83 + {
84 + const int idx = thread_row_offset + ii;
85 + threadData += exp((static_cast<float>(input[idx]) - float_max));
86 + }
87 +
88 + const auto Z = BlockReduce(tmpStorage).Reduce(threadData, sum);
89 +
90 + if (threadIdx.x == 0)
91 + {
92 + normalizing_factor = 1.f / Z;
93 + }
94 + __syncthreads();
95 +
96 + for (int ii = threadIdx.x; ii < num_cols; ii += TPB)
97 + {
98 + const int idx = thread_row_offset + ii;
99 + const float val = exp((static_cast<float>(input[idx]) - float_max)) * normalizing_factor;
100 + output[idx] = val;
101 + }
102 +}
103 +
104 +template <int TPB>
105 +__launch_bounds__(TPB) __global__ void moeTopK(const float* inputs_after_softmax, const bool* finished, float* output,
106 + int* indices, int* source_rows, const int num_experts, const int k, const int start_expert, const int end_expert)
107 +{
108 +
109 + using cub_kvp = cub::KeyValuePair<int, float>;
110 + using BlockReduce = cub::BlockReduce<cub_kvp, TPB>;
111 + __shared__ typename BlockReduce::TempStorage tmpStorage;
112 +
113 + cub_kvp thread_kvp;
114 + cub::ArgMax arg_max;
115 +
116 + const int num_rows = gridDim.x;
117 + const int block_row = blockIdx.x;
118 +
119 + const bool row_is_active = finished ? !finished[block_row] : true;
120 + const int thread_read_offset = blockIdx.x * num_experts;
121 + for (int k_idx = 0; k_idx < k; ++k_idx)
122 + {
123 + thread_kvp.key = 0;
124 + thread_kvp.value = -1.f; // This is OK because inputs are probabilities
125 +
126 + cub_kvp inp_kvp;
127 + for (int expert = threadIdx.x; expert < num_experts; expert += TPB)
128 + {
129 + const int idx = thread_read_offset + expert;
130 + inp_kvp.key = expert;
131 + inp_kvp.value = inputs_after_softmax[idx];
132 +
133 + for (int prior_k = 0; prior_k < k_idx; ++prior_k)
134 + {
135 + const int prior_winning_expert = indices[k * block_row + prior_k];
136 +
137 + if (prior_winning_expert == expert)
138 + {
139 + inp_kvp = thread_kvp;
140 + }
141 + }
142 +
143 + thread_kvp = arg_max(inp_kvp, thread_kvp);
144 + }
145 +
146 + const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max);
147 + if (threadIdx.x == 0)
148 + {
149 + // Ignore experts the node isn't responsible for with expert parallelism
150 + const int expert = result_kvp.key;
151 + const bool node_uses_expert = expert >= start_expert && expert < end_expert;
152 + const bool should_process_row = row_is_active && node_uses_expert;
153 +
154 + const int idx = k * block_row + k_idx;
155 + output[idx] = result_kvp.value;
156 + indices[idx] = should_process_row ? (expert - start_expert) : num_experts;
157 + assert(indices[idx] >= 0);
158 + source_rows[idx] = k_idx * num_rows + block_row;
159 + }
160 + __syncthreads();
161 + }
162 +}
163 +
164 +// ====================== TopK softmax things ===============================
165 +
166 +/*
167 + A Top-K gating softmax written to exploit when the number of experts in the MoE layers
168 + are a small power of 2. This allows us to cleanly share the rows among the threads in
169 + a single warp and eliminate communication between warps (so no need to use shared mem).
170 +
171 + It fuses the softmax, max and argmax into a single kernel.
172 +
173 + Limitations:
174 + 1) This implementation is intended for when the number of experts is a small power of 2.
175 + 2) This implementation assumes k is small, but will work for any k.
176 +*/
177 +
178 +template <int VPT, int NUM_EXPERTS, int WARPS_PER_CTA, int BYTES_PER_LDG>
179 +__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__
180 + void topkGatingSoftmax(const float* input, const bool* finished, float* output, const int num_rows, int* indices,
181 + int* source_rows, const int k, const int start_expert, const int end_expert)
182 +{
183 + // We begin by enforcing compile time assertions and setting up compile time constants.
184 + static_assert(VPT == (VPT & -VPT), "VPT must be power of 2");
185 + static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2");
186 + static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2");
187 + static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16");
188 +
189 + // Number of bytes each thread pulls in per load
190 + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(float);
191 + static constexpr int ELTS_PER_ROW = NUM_EXPERTS;
192 + static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT;
193 + static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG;
194 +
195 + // Restrictions based on previous section.
196 + static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg");
197 + static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp");
198 + static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2");
199 + static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size");
200 +
201 + // We have NUM_EXPERTS elements per row. We specialize for small #experts
202 + static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT;
203 + static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW;
204 + static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP;
205 +
206 + // Restrictions for previous section.
207 + static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp");
208 +
209 + // ===================== From this point, we finally start computing run-time variables. ========================
210 +
211 + // Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps.
212 + // This, each block processes a chunk of rows. We start by computing the start row for each block.
213 + const int cta_base_row = blockIdx.x * ROWS_PER_CTA;
214 +
215 + // Now, using the base row per thread block, we compute the base row per warp.
216 + const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP;
217 +
218 + // The threads in a warp are split into sub-groups that will work on a row.
219 + // We compute row offset for each thread sub-group
220 + const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW;
221 + const int thread_row = warp_base_row + thread_row_in_warp;
222 +
223 + // Threads with indices out of bounds should early exit here.
224 + if (thread_row >= num_rows)
225 + {
226 + return;
227 + }
228 + const bool row_is_active = finished ? !finished[thread_row] : true;
229 +
230 + // We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the
231 + // row it will read.
232 + const float* thread_row_ptr = input + thread_row * ELTS_PER_ROW;
233 +
234 + // Now, we compute the group each thread belong to in order to determine the first column to start loads.
235 + const int thread_group_idx = threadIdx.x % THREADS_PER_ROW;
236 + const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG;
237 + const float* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread;
238 +
239 + // Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory,
240 + // this can support all powers of 2 up to 16.
241 + // NOTE(woosuk): The original implementation uses CUTLASS aligned array here.
242 + // We defined our own aligned array and use it here to avoid the dependency on CUTLASS.
243 + using AccessType = AlignedArray<float, ELTS_PER_LDG>;
244 +
245 + // Finally, we pull in the data from global mem
246 + float row_chunk[VPT];
247 + AccessType* row_chunk_vec_ptr = reinterpret_cast<AccessType*>(&row_chunk);
248 + const AccessType* vec_thread_read_ptr = reinterpret_cast<const AccessType*>(thread_read_ptr);
249 +#pragma unroll
250 + for (int ii = 0; ii < LDG_PER_THREAD; ++ii)
251 + {
252 + row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW];
253 + }
254 +
255 + // First, we perform a max reduce within the thread. We can do the max in fp16 safely (I think) and just
256 + // convert to float afterwards for the exp + sum reduction.
257 + float thread_max = row_chunk[0];
258 +#pragma unroll
259 + for (int ii = 1; ii < VPT; ++ii)
260 + {
261 + thread_max = max(thread_max, row_chunk[ii]);
262 + }
263 +
264 +// Now, we find the max within the thread group and distribute among the threads. We use a butterfly reduce.
265 +#pragma unroll
266 + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
267 + {
268 + thread_max = max(thread_max, __shfl_xor_sync(0xFFFFFFFF, thread_max, mask, THREADS_PER_ROW));
269 + }
270 +
271 + // From this point, thread max in all the threads have the max within the row.
272 + // Now, we subtract the max from each element in the thread and take the exp. We also compute the thread local sum.
273 + float row_sum = 0;
274 +#pragma unroll
275 + for (int ii = 0; ii < VPT; ++ii)
276 + {
277 + row_chunk[ii] = expf(row_chunk[ii] - thread_max);
278 + row_sum += row_chunk[ii];
279 + }
280 +
281 +// Now, we perform the sum reduce within each thread group. Similar to the max reduce, we use a bufferfly pattern.
282 +#pragma unroll
283 + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
284 + {
285 + row_sum += __shfl_xor_sync(0xFFFFFFFF, row_sum, mask, THREADS_PER_ROW);
286 + }
287 +
288 + // From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables
289 + // respectively. Finally, we can scale the rows for the softmax. Technically, for top-k gating we don't need to
290 + // compute the entire softmax row. We can likely look at the maxes and only compute for the top-k values in the row.
291 + // However, this kernel will likely not be a bottle neck and it seems better to closer match torch and find the
292 + // argmax after computing the softmax.
293 + const float reciprocal_row_sum = 1.f / row_sum;
294 +
295 +#pragma unroll
296 + for (int ii = 0; ii < VPT; ++ii)
297 + {
298 + row_chunk[ii] = row_chunk[ii] * reciprocal_row_sum;
299 + }
300 +
301 + // Now, softmax_res contains the softmax of the row chunk. Now, I want to find the topk elements in each row, along
302 + // with the max index.
303 + int start_col = first_elt_read_by_thread;
304 + static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW;
305 +
306 + for (int k_idx = 0; k_idx < k; ++k_idx)
307 + {
308 + // First, each thread does the local argmax
309 + float max_val = row_chunk[0];
310 + int expert = start_col;
311 +#pragma unroll
312 + for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG)
313 + {
314 +#pragma unroll
315 + for (int ii = 0; ii < ELTS_PER_LDG; ++ii)
316 + {
317 + float val = row_chunk[ldg * ELTS_PER_LDG + ii];
318 +
319 + // No check on the experts here since columns with the smallest index are processed first and only
320 + // updated if > (not >=)
321 + if (val > max_val)
322 + {
323 + max_val = val;
324 + expert = col + ii;
325 + }
326 + }
327 + }
328 +
329 +// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max.
330 +// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can
331 +// then blank out their max with -inf and the warp can run more iterations...
332 +#pragma unroll
333 + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
334 + {
335 + float other_max = __shfl_xor_sync(0xFFFFFFFF, max_val, mask, THREADS_PER_ROW);
336 + int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, THREADS_PER_ROW);
337 +
338 + // We want lower indices to "win" in every thread so we break ties this way
339 + if (other_max > max_val || (other_max == max_val && other_expert < expert))
340 + {
341 + max_val = other_max;
342 + expert = other_expert;
343 + }
344 + }
345 +
346 + // Write the max for this k iteration to global memory.
347 + if (thread_group_idx == 0)
348 + {
349 + // Add a guard to ignore experts not included by this node
350 + const bool node_uses_expert = expert >= start_expert && expert < end_expert;
351 + const bool should_process_row = row_is_active && node_uses_expert;
352 +
353 + // The lead thread from each sub-group will write out the final results to global memory. (This will be a
354 + // single) thread per row of the input/output matrices.
355 + const int idx = k * thread_row + k_idx;
356 + output[idx] = max_val;
357 + indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS;
358 + source_rows[idx] = k_idx * num_rows + thread_row;
359 + }
360 +
361 + // Finally, we clear the value in the thread with the current max if there is another iteration to run.
362 + if (k_idx + 1 < k)
363 + {
364 + const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG;
365 + const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW;
366 +
367 + // Only the thread in the group which produced the max will reset the "winning" value to -inf.
368 + if (thread_group_idx == thread_to_clear_in_group)
369 + {
370 + const int offset_for_expert = expert % ELTS_PER_LDG;
371 + // Safe to set to any negative value since row_chunk values must be between 0 and 1.
372 + row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f;
373 + }
374 + }
375 + }
376 +}
377 +
378 +namespace detail
379 +{
380 +// Constructs some constants needed to partition the work across threads at compile time.
381 +template <int EXPERTS, int BYTES_PER_LDG>
382 +struct TopkConstants
383 +{
384 + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(float);
385 + static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, "");
386 + static constexpr int VECs_PER_THREAD = std::max(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE));
387 + static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG;
388 + static constexpr int THREADS_PER_ROW = EXPERTS / VPT;
389 + static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW;
390 +};
391 +} // namespace detail
392 +
393 +template <int EXPERTS, int WARPS_PER_TB>
394 +void topkGatingSoftmaxLauncherHelper(const float* input, const bool* finished, float* output, int* indices,
395 + int* source_row, const int num_rows, const int k, const int start_expert, const int end_expert, cudaStream_t stream)
396 +{
397 + static constexpr std::size_t MAX_BYTES_PER_LDG = 16;
398 +
399 + static constexpr int BYTES_PER_LDG = std::min(MAX_BYTES_PER_LDG, sizeof(float) * EXPERTS);
400 + using Constants = detail::TopkConstants<EXPERTS, BYTES_PER_LDG>;
401 + static constexpr int VPT = Constants::VPT;
402 + static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP;
403 + const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP;
404 + const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB;
405 +
406 + dim3 block_dim(WARP_SIZE, WARPS_PER_TB);
407 + topkGatingSoftmax<VPT, EXPERTS, WARPS_PER_TB, BYTES_PER_LDG><<<num_blocks, block_dim, 0, stream>>>(
408 + input, finished, output, num_rows, indices, source_row, k, start_expert, end_expert);
409 +}
410 +
411 +#define LAUNCH_SOFTMAX(NUM_EXPERTS, WARPS_PER_TB) \
412 + topkGatingSoftmaxLauncherHelper<NUM_EXPERTS, WARPS_PER_TB>( \
413 + gating_output, nullptr, topk_weights, topk_indicies, \
414 + token_expert_indices, num_tokens, topk, 0, num_experts, \
415 + stream);
416 +
417 +void topkGatingSoftmaxKernelLauncher(
418 + const float* gating_output,
419 + float* topk_weights,
420 + int* topk_indicies,
421 + int* token_expert_indices,
422 + float* softmax_workspace,
423 + const int num_tokens,
424 + const int num_experts,
425 + const int topk,
426 + cudaStream_t stream) {
427 + static constexpr int WARPS_PER_TB = 4;
428 + switch (num_experts) {
429 + case 1:
430 + LAUNCH_SOFTMAX(1, WARPS_PER_TB);
431 + break;
432 + case 2:
433 + LAUNCH_SOFTMAX(2, WARPS_PER_TB);
434 + break;
435 + case 4:
436 + LAUNCH_SOFTMAX(4, WARPS_PER_TB);
437 + break;
438 + case 8:
439 + LAUNCH_SOFTMAX(8, WARPS_PER_TB);
440 + break;
441 + case 16:
442 + LAUNCH_SOFTMAX(16, WARPS_PER_TB);
443 + break;
444 + case 32:
445 + LAUNCH_SOFTMAX(32, WARPS_PER_TB);
446 + break;
447 + case 64:
448 + LAUNCH_SOFTMAX(64, WARPS_PER_TB);
449 + break;
450 + case 128:
451 + LAUNCH_SOFTMAX(128, WARPS_PER_TB);
452 + break;
453 + case 256:
454 + LAUNCH_SOFTMAX(256, WARPS_PER_TB);
455 + break;
456 + default: {
457 + TORCH_CHECK(softmax_workspace != nullptr,
458 + "softmax_workspace must be provided for num_experts that are not a power of 2.");
459 + static constexpr int TPB = 256;
460 + moeSoftmax<TPB><<<num_tokens, TPB, 0, stream>>>(
461 + gating_output, nullptr, softmax_workspace, num_experts);
462 + moeTopK<TPB><<<num_tokens, TPB, 0, stream>>>(
463 + softmax_workspace, nullptr, topk_weights, topk_indicies, token_expert_indices,
464 + num_experts, topk, 0, num_experts);
465 + }
466 + }
467 +}
468 +
469 +} // namespace moe
470 +} // namespace vllm
471 +
472 +void topk_softmax(
473 + torch::Tensor& topk_weights, // [num_tokens, topk]
474 + torch::Tensor& topk_indices, // [num_tokens, topk]
475 + torch::Tensor& token_expert_indices, // [num_tokens, topk]
476 + torch::Tensor& gating_output) // [num_tokens, num_experts]
477 +{
478 + const int num_experts = gating_output.size(-1);
479 + const int num_tokens = gating_output.numel() / num_experts;
480 + const int topk = topk_weights.size(-1);
481 +
482 + const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0);
483 + const bool needs_workspace = !is_pow_2 || num_experts > 256;
484 + const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0;
485 +
486 + const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output));
487 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
488 + torch::Tensor softmax_workspace = torch::empty({workspace_size}, gating_output.options());
489 + vllm::moe::topkGatingSoftmaxKernelLauncher(
490 + gating_output.data_ptr<float>(),
491 + topk_weights.data_ptr<float>(),
492 + topk_indices.data_ptr<int>(),
493 + token_expert_indices.data_ptr<int>(),
494 + softmax_workspace.data_ptr<float>(),
495 + num_tokens,
496 + num_experts,
497 + topk,
498 + stream);
499 +}
csrc/pybind.cpp
+1 -1
@@ -48,8 +48,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
48 &rotary_embedding,
49 "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
50
51 -#ifndef USE_ROCM
51 // Quantization ops
52 +#ifndef USE_ROCM
53 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
54 ops.def("awq_dequantize", &awq_dequantize, "Dequantization for AWQ");
55 #endif
setup.py
+11
@@ -339,6 +339,17 @@ if _is_cuda():
339 vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
340 vllm_extension_sources.append("csrc/custom_all_reduce.cu")
341
342 + # Add MoE kernels.
343 + ext_modules.append(
344 + CUDAExtension(
345 + name="vllm._moe_C",
346 + sources=glob("csrc/moe/*.cu") + glob("csrc/moe/*.cpp"),
347 + extra_compile_args={
348 + "cxx": CXX_FLAGS,
349 + "nvcc": NVCC_FLAGS,
350 + },
351 + ))
352 +
353 if not _is_neuron():
354 vllm_extension = CUDAExtension(
355 name="vllm._C",
tests/kernels/test_moe.py
+10 -16
@@ -2,10 +2,8 @@
2
3 Run `pytest tests/kernels/test_moe.py`.
4 """
5 -
5 import pytest
6 import torch
8 -
7 from transformers import MixtralConfig
8 from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
9
@@ -14,22 +12,21 @@ from vllm.model_executor.layers.activation import SiluAndMul
12 from vllm.model_executor.models.mixtral import MixtralMoE
13
14
17 -def torch_moe(a, w1, w2, topk_weight, topk_ids):
15 +def torch_moe(a, w1, w2, score, topk):
16 B, D = a.shape
19 - a = a.view(B, -1, D).repeat(1, topk_ids.shape[1], 1).reshape(-1, D)
20 - out = torch.zeros(B * topk_ids.shape[1],
21 - w2.shape[1],
22 - dtype=a.dtype,
23 - device=a.device)
24 - topk_ids = topk_ids.view(-1)
17 + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
18 + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
19 + score = torch.softmax(score, dim=-1, dtype=torch.float32)
20 + topk_weight, topk_ids = torch.topk(score, topk)
21 topk_weight = topk_weight.view(-1)
22 + topk_ids = topk_ids.view(-1)
23 for i in range(w1.shape[0]):
24 mask = topk_ids == i
25 if mask.sum():
26 out[mask] = SiluAndMul()(
27 a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose(0, 1)
28 return (out.view(B, -1, w2.shape[1]) *
32 - topk_weight.view(B, -1, 1)).sum(dim=1)
29 + topk_weight.view(B, -1, 1).to(out.dtype)).sum(dim=1)
30
31
32 @pytest.mark.parametrize("m", [512, 222, 33, 1])
@@ -51,11 +48,8 @@ def test_fused_moe(
48 w2 = torch.randn((e, k, n), device='cuda', dtype=dtype) / 10
49
50 score = torch.randn((m, e), device='cuda', dtype=dtype)
54 - score = torch.softmax(score, dim=-1)
55 - topk_weight, topk_ids = torch.topk(score, topk)
56 -
57 - triton_output = fused_moe(a, w1, w2, topk_weight, topk_ids, False)
58 - torch_output = torch_moe(a, w1, w2, topk_weight, topk_ids)
51 + triton_output = fused_moe(a, w1, w2, score, topk, renormalize=False)
52 + torch_output = torch_moe(a, w1, w2, score, topk)
53 assert torch.allclose(triton_output, torch_output, atol=1e-2, rtol=0)
54
55
@@ -75,7 +69,7 @@ def test_mixtral_moe(dtype: torch.dtype):
69 intermediate_size=config.intermediate_size,
70 params_dtype=dtype,
71 tp_size=1,
78 - )
72 + ).cuda()
73
74 # Load the weights
75 vllm_moe.gate.linear_weights["weight"][:] = hf_moe.gate.weight.data
vllm/model_executor/layers/fused_moe.py
+48 -10
@@ -4,6 +4,7 @@ import triton
4 import triton.language as tl
5
6 from vllm._C import ops
7 +from vllm.utils import is_hip
8
9
10 @triton.jit
@@ -177,7 +178,6 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
178 expert_ids: torch.Tensor,
179 num_tokens_post_padded: torch.Tensor,
180 mul_routed_weight: bool, top_k: int, config: dict):
180 -
181 assert topk_weights.stride(1) == 1
182 assert sorted_token_ids.stride(0) == 1
183
@@ -210,12 +210,15 @@ def invoke_fused_moe_kernel(A: torch.Tensor, B: torch.Tensor, C: torch.Tensor,
210 )
211
212
213 -def fused_moe(hidden_states: torch.Tensor,
214 - w1: torch.Tensor,
215 - w2: torch.Tensor,
216 - topk_weights: torch.Tensor,
217 - topk_ids: torch.Tensor,
218 - inplace=False):
213 +def fused_moe(
214 + hidden_states: torch.Tensor,
215 + w1: torch.Tensor,
216 + w2: torch.Tensor,
217 + gating_output: torch.Tensor,
218 + topk: int,
219 + renormalize: bool,
220 + inplace: bool = False,
221 +) -> torch.Tensor:
222 """
223 This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism.
224
@@ -223,15 +226,19 @@ def fused_moe(hidden_states: torch.Tensor,
226 - hidden_states (torch.Tensor): The input tensor to the MoE layer.
227 - w1 (torch.Tensor): The first set of expert weights.
228 - w2 (torch.Tensor): The second set of expert weights.
226 - - topk_weights (torch.Tensor): The weights for the top-k selected experts.
227 - - topk_ids (torch.Tensor): The indices of the top-k selected experts.
229 + - gating_output (torch.Tensor): The output of the gating operation (before softmax).
230 + - topk (int): The number of top-k experts to select.
231 + - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
232 - inplace (bool): If True, perform the operation in-place. Defaults to False.
233
234 Returns:
235 - torch.Tensor: The output tensor after applying the MoE layer.
236 """
237 # Check constraints.
234 - assert hidden_states.shape[1] == w1.shape[2], "Incompatible dimensions"
238 + assert hidden_states.shape[0] == gating_output.shape[0], (
239 + "Number of tokens mismatch")
240 + assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
241 + assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
242 assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
243 assert w1.is_contiguous(), "Expert weights1 must be contiguous"
244 assert w2.is_contiguous(), "Expert weights2 must be contiguous"
@@ -241,6 +248,37 @@ def fused_moe(hidden_states: torch.Tensor,
248 M, _ = hidden_states.shape
249 E, N, _ = w1.shape
250
251 + if is_hip():
252 + # The MoE kernels are not yet supported on ROCm.
253 + routing_weights = torch.softmax(gating_output,
254 + dim=-1,
255 + dtype=torch.float32)
256 + topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
257 + else:
258 + import vllm._moe_C as moe_kernels
259 +
260 + topk_weights = torch.empty(M,
261 + topk,
262 + dtype=torch.float32,
263 + device=hidden_states.device)
264 + topk_ids = torch.empty(M,
265 + topk,
266 + dtype=torch.int32,
267 + device=hidden_states.device)
268 + token_expert_indicies = torch.empty(M,
269 + topk,
270 + dtype=torch.int32,
271 + device=hidden_states.device)
272 + moe_kernels.topk_softmax(
273 + topk_weights,
274 + topk_ids,
275 + token_expert_indicies,
276 + gating_output.float(), # TODO(woosuk): Optimize this.
277 + )
278 + del token_expert_indicies # Not used. Will be used in the future.
279 + if renormalize:
280 + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
281 +
282 config = {
283 'BLOCK_SIZE_M': 64,
284 'BLOCK_SIZE_N': 64,
vllm/model_executor/models/deepseek.py
+3 -12
@@ -25,7 +25,6 @@ from typing import Any, Dict, List, Optional, Tuple
25
26 import torch
27 from torch import nn
28 -import torch.nn.functional as F
28 from transformers import PretrainedConfig
29
30 from vllm.model_executor.input_metadata import InputMetadata
@@ -155,20 +154,12 @@ class DeepseekMoE(nn.Module):
154 shared_output = self.shared_experts(hidden_states)
155 # router_logits: (batch * sequence_length, n_experts)
156 router_logits, _ = self.gate(hidden_states)
158 -
159 - routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
160 - routing_weights, selected_experts = torch.topk(routing_weights,
161 - self.top_k,
162 - dim=-1)
163 -
164 - if self.config.norm_topk_prob:
165 - routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
166 -
157 final_hidden_states = fused_moe(hidden_states,
158 self.w1,
159 self.w2,
170 - routing_weights,
171 - selected_experts,
160 + router_logits,
161 + self.top_k,
162 + renormalize=self.config.norm_topk_prob,
163 inplace=True)
164
165 if self.config.n_shared_experts is not None:
vllm/model_executor/models/mixtral.py
+3 -11
@@ -24,8 +24,6 @@
24 from typing import List, Optional, Tuple
25
26 import torch
27 -import torch.nn.functional as F
28 -
27 from torch import nn
28 from transformers import MixtralConfig
29
@@ -128,18 +126,12 @@ class MixtralMoE(nn.Module):
126 hidden_states = hidden_states.view(-1, self.hidden_size)
127 # router_logits: (batch * sequence_length, n_experts)
128 router_logits, _ = self.gate(hidden_states)
131 -
132 - routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
133 - routing_weights, selected_experts = torch.topk(routing_weights,
134 - self.top_k,
135 - dim=-1)
136 - routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
137 -
129 final_hidden_states = fused_moe(hidden_states,
130 self.ws,
131 self.w2s,
141 - routing_weights,
142 - selected_experts,
132 + router_logits,
133 + self.top_k,
134 + renormalize=True,
135 inplace=True)
136
137 if self.tp_size > 1: