A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel][ROCm][AMD] enable fused topk_softmax kernel for moe layer (#4927)
This PR enables the fused topk_softmax kernel used in moe layer for HIP
Divakar Verma committed
Jun 2, 2024 at 16:13 UTC
a66cf40b205d57ac1b5dc96b6bb6f8e813b18316
6 files changed
+45
-43
CMakeLists.txt
+3
-5
@@ -311,6 +311,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
311
message(STATUS "Enabling C extension.")
312
add_dependencies(default _C)
313
314
+ message(STATUS "Enabling moe extension.")
315
+ add_dependencies(default _moe_C)
316
+
317
# Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
318
# VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
319
# there are supported target arches.
@@ -320,8 +323,3 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
323
add_dependencies(default _punica_C)
324
endif()
325
endif()
323
-
324
-if(VLLM_GPU_LANG STREQUAL "CUDA")
325
- message(STATUS "Enabling moe extension.")
326
- add_dependencies(default _moe_C)
327
-endif()
Dockerfile.rocm
+1
@@ -108,6 +108,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
108
&& python3 setup.py install \
109
&& cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
110
&& cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.cpython-39-x86_64-linux-gnu.so vllm/ \
111
+ && cp build/lib.linux-x86_64-cpython-39/vllm/_moe_C.cpython-39-x86_64-linux-gnu.so vllm/ \
112
&& cd ..
113
114
csrc/cuda_compat.h
+4
@@ -19,8 +19,12 @@
19
#ifndef USE_ROCM
20
#define VLLM_SHFL_XOR_SYNC(var, lane_mask) \
21
__shfl_xor_sync(uint32_t(-1), var, lane_mask)
22
+ #define VLLM_SHFL_XOR_SYNC_WIDTH(var, lane_mask, width) \
23
+ __shfl_xor_sync(uint32_t(-1), var, lane_mask, width)
24
#else
25
#define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor(var, lane_mask)
26
+ #define VLLM_SHFL_XOR_SYNC_WIDTH(var, lane_mask, width) \
27
+ __shfl_xor(var, lane_mask, width)
28
#endif
29
30
#ifndef USE_ROCM
csrc/moe/topk_softmax_kernels.cu
+17
-10
@@ -19,15 +19,22 @@
19
#include <torch/extension.h>
20
#include <ATen/cuda/CUDAContext.h>
21
#include <c10/cuda/CUDAGuard.h>
22
+#include "../cuda_compat.h"
23
23
-#include <cub/cub.cuh>
24
-#include <cub/util_type.cuh>
24
+#ifndef USE_ROCM
25
+ #include <cub/util_type.cuh>
26
+ #include <cub/cub.cuh>
27
+#else
28
+ #include <hipcub/util_type.hpp>
29
+ #include <hipcub/hipcub.hpp>
30
+#endif
31
+
32
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
33
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
34
35
namespace vllm {
36
namespace moe {
37
29
-static constexpr int WARP_SIZE = 32;
30
-
38
/// Aligned array type
39
template <
40
typename T,
@@ -265,7 +272,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__
272
#pragma unroll
273
for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
274
{
268
- thread_max = max(thread_max, __shfl_xor_sync(0xFFFFFFFF, thread_max, mask, THREADS_PER_ROW));
275
+ thread_max = max(thread_max, VLLM_SHFL_XOR_SYNC_WIDTH(thread_max, mask, THREADS_PER_ROW));
276
}
277
278
// From this point, thread max in all the threads have the max within the row.
@@ -282,7 +289,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__
289
#pragma unroll
290
for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
291
{
285
- row_sum += __shfl_xor_sync(0xFFFFFFFF, row_sum, mask, THREADS_PER_ROW);
292
+ row_sum += VLLM_SHFL_XOR_SYNC_WIDTH(row_sum, mask, THREADS_PER_ROW);
293
}
294
295
// From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables
@@ -332,8 +339,8 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__
339
#pragma unroll
340
for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
341
{
335
- float other_max = __shfl_xor_sync(0xFFFFFFFF, max_val, mask, THREADS_PER_ROW);
336
- int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, THREADS_PER_ROW);
342
+ float other_max = VLLM_SHFL_XOR_SYNC_WIDTH(max_val, mask, THREADS_PER_ROW);
343
+ int other_expert = VLLM_SHFL_XOR_SYNC_WIDTH(expert, mask, THREADS_PER_ROW);
344
345
// We want lower indices to "win" in every thread so we break ties this way
346
if (other_max > max_val || (other_max == max_val && other_expert < expert))
@@ -383,7 +390,7 @@ struct TopkConstants
390
{
391
static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(float);
392
static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, "");
386
- static constexpr int VECs_PER_THREAD = std::max(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE));
393
+ static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE));
394
static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG;
395
static constexpr int THREADS_PER_ROW = EXPERTS / VPT;
396
static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW;
@@ -396,7 +403,7 @@ void topkGatingSoftmaxLauncherHelper(const float* input, const bool* finished, f
403
{
404
static constexpr std::size_t MAX_BYTES_PER_LDG = 16;
405
399
- static constexpr int BYTES_PER_LDG = std::min(MAX_BYTES_PER_LDG, sizeof(float) * EXPERTS);
406
+ static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(float) * EXPERTS);
407
using Constants = detail::TopkConstants<EXPERTS, BYTES_PER_LDG>;
408
static constexpr int VPT = Constants::VPT;
409
static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP;
setup.py
+1
-1
@@ -382,7 +382,7 @@ def get_requirements() -> List[str]:
382
383
ext_modules = []
384
385
-if _is_cuda():
385
+if _is_cuda() or _is_hip():
386
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
387
388
if not _is_neuron():
vllm/model_executor/layers/fused_moe/fused_moe.py
+19
-27
@@ -8,9 +8,9 @@ import torch
8
import triton
9
import triton.language as tl
10
11
+import vllm._moe_C as moe_kernels
12
from vllm import _custom_ops as ops
13
from vllm.logger import init_logger
13
-from vllm.utils import is_hip
14
15
logger = init_logger(__name__)
16
@@ -319,34 +319,26 @@ def fused_topk(
319
320
M, _ = hidden_states.shape
321
322
- if is_hip():
323
- # The MoE kernels are not yet supported on ROCm.
324
- routing_weights = torch.softmax(gating_output,
325
- dim=-1,
326
- dtype=torch.float32)
327
- topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
328
- else:
329
- import vllm._moe_C as moe_kernels
330
-
331
- topk_weights = torch.empty(M,
332
- topk,
333
- dtype=torch.float32,
334
- device=hidden_states.device)
335
- topk_ids = torch.empty(M,
322
+ topk_weights = torch.empty(M,
323
topk,
337
- dtype=torch.int32,
324
+ dtype=torch.float32,
325
device=hidden_states.device)
339
- token_expert_indicies = torch.empty(M,
340
- topk,
341
- dtype=torch.int32,
342
- device=hidden_states.device)
343
- moe_kernels.topk_softmax(
344
- topk_weights,
345
- topk_ids,
346
- token_expert_indicies,
347
- gating_output.float(), # TODO(woosuk): Optimize this.
348
- )
349
- del token_expert_indicies # Not used. Will be used in the future.
326
+ topk_ids = torch.empty(M,
327
+ topk,
328
+ dtype=torch.int32,
329
+ device=hidden_states.device)
330
+ token_expert_indicies = torch.empty(M,
331
+ topk,
332
+ dtype=torch.int32,
333
+ device=hidden_states.device)
334
+ moe_kernels.topk_softmax(
335
+ topk_weights,
336
+ topk_ids,
337
+ token_expert_indicies,
338
+ gating_output.float(), # TODO(woosuk): Optimize this.
339
+ )
340
+ del token_expert_indicies # Not used. Will be used in the future.
341
+
342
if renormalize:
343
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
344
return topk_weights, topk_ids