@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a66cf40b20
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel][ROCm][AMD] enable fused topk_softmax kernel for moe layer (#4927)

This PR enables the fused topk_softmax kernel used in moe layer for HIP

Divakar Verma committed Jun 2, 2024 at 16:13 UTC a66cf40b205d57ac1b5dc96b6bb6f8e813b18316
6 files changed +45 -43
CMakeLists.txt
+3 -5
@@ -311,6 +311,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
311 message(STATUS "Enabling C extension.")
312 add_dependencies(default _C)
313
314 + message(STATUS "Enabling moe extension.")
315 + add_dependencies(default _moe_C)
316 +
317 # Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
318 # VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
319 # there are supported target arches.
@@ -320,8 +323,3 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
323 add_dependencies(default _punica_C)
324 endif()
325 endif()
323 -
324 -if(VLLM_GPU_LANG STREQUAL "CUDA")
325 - message(STATUS "Enabling moe extension.")
326 - add_dependencies(default _moe_C)
327 -endif()
Dockerfile.rocm
+1
@@ -108,6 +108,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
108 && python3 setup.py install \
109 && cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
110 && cp build/lib.linux-x86_64-cpython-39/vllm/_punica_C.cpython-39-x86_64-linux-gnu.so vllm/ \
111 + && cp build/lib.linux-x86_64-cpython-39/vllm/_moe_C.cpython-39-x86_64-linux-gnu.so vllm/ \
112 && cd ..
113
114
csrc/cuda_compat.h
+4
@@ -19,8 +19,12 @@
19 #ifndef USE_ROCM
20 #define VLLM_SHFL_XOR_SYNC(var, lane_mask) \
21 __shfl_xor_sync(uint32_t(-1), var, lane_mask)
22 + #define VLLM_SHFL_XOR_SYNC_WIDTH(var, lane_mask, width) \
23 + __shfl_xor_sync(uint32_t(-1), var, lane_mask, width)
24 #else
25 #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor(var, lane_mask)
26 + #define VLLM_SHFL_XOR_SYNC_WIDTH(var, lane_mask, width) \
27 + __shfl_xor(var, lane_mask, width)
28 #endif
29
30 #ifndef USE_ROCM
csrc/moe/topk_softmax_kernels.cu
+17 -10
@@ -19,15 +19,22 @@
19 #include <torch/extension.h>
20 #include <ATen/cuda/CUDAContext.h>
21 #include <c10/cuda/CUDAGuard.h>
22 +#include "../cuda_compat.h"
23
23 -#include <cub/cub.cuh>
24 -#include <cub/util_type.cuh>
24 +#ifndef USE_ROCM
25 + #include <cub/util_type.cuh>
26 + #include <cub/cub.cuh>
27 +#else
28 + #include <hipcub/util_type.hpp>
29 + #include <hipcub/hipcub.hpp>
30 +#endif
31 +
32 +#define MAX(a, b) ((a) > (b) ? (a) : (b))
33 +#define MIN(a, b) ((a) < (b) ? (a) : (b))
34
35 namespace vllm {
36 namespace moe {
37
29 -static constexpr int WARP_SIZE = 32;
30 -
38 /// Aligned array type
39 template <
40 typename T,
@@ -265,7 +272,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__
272 #pragma unroll
273 for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
274 {
268 - thread_max = max(thread_max, __shfl_xor_sync(0xFFFFFFFF, thread_max, mask, THREADS_PER_ROW));
275 + thread_max = max(thread_max, VLLM_SHFL_XOR_SYNC_WIDTH(thread_max, mask, THREADS_PER_ROW));
276 }
277
278 // From this point, thread max in all the threads have the max within the row.
@@ -282,7 +289,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__
289 #pragma unroll
290 for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
291 {
285 - row_sum += __shfl_xor_sync(0xFFFFFFFF, row_sum, mask, THREADS_PER_ROW);
292 + row_sum += VLLM_SHFL_XOR_SYNC_WIDTH(row_sum, mask, THREADS_PER_ROW);
293 }
294
295 // From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables
@@ -332,8 +339,8 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__
339 #pragma unroll
340 for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2)
341 {
335 - float other_max = __shfl_xor_sync(0xFFFFFFFF, max_val, mask, THREADS_PER_ROW);
336 - int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, THREADS_PER_ROW);
342 + float other_max = VLLM_SHFL_XOR_SYNC_WIDTH(max_val, mask, THREADS_PER_ROW);
343 + int other_expert = VLLM_SHFL_XOR_SYNC_WIDTH(expert, mask, THREADS_PER_ROW);
344
345 // We want lower indices to "win" in every thread so we break ties this way
346 if (other_max > max_val || (other_max == max_val && other_expert < expert))
@@ -383,7 +390,7 @@ struct TopkConstants
390 {
391 static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(float);
392 static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, "");
386 - static constexpr int VECs_PER_THREAD = std::max(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE));
393 + static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE));
394 static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG;
395 static constexpr int THREADS_PER_ROW = EXPERTS / VPT;
396 static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW;
@@ -396,7 +403,7 @@ void topkGatingSoftmaxLauncherHelper(const float* input, const bool* finished, f
403 {
404 static constexpr std::size_t MAX_BYTES_PER_LDG = 16;
405
399 - static constexpr int BYTES_PER_LDG = std::min(MAX_BYTES_PER_LDG, sizeof(float) * EXPERTS);
406 + static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(float) * EXPERTS);
407 using Constants = detail::TopkConstants<EXPERTS, BYTES_PER_LDG>;
408 static constexpr int VPT = Constants::VPT;
409 static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP;
setup.py
+1 -1
@@ -382,7 +382,7 @@ def get_requirements() -> List[str]:
382
383 ext_modules = []
384
385 -if _is_cuda():
385 +if _is_cuda() or _is_hip():
386 ext_modules.append(CMakeExtension(name="vllm._moe_C"))
387
388 if not _is_neuron():
vllm/model_executor/layers/fused_moe/fused_moe.py
+19 -27
@@ -8,9 +8,9 @@ import torch
8 import triton
9 import triton.language as tl
10
11 +import vllm._moe_C as moe_kernels
12 from vllm import _custom_ops as ops
13 from vllm.logger import init_logger
13 -from vllm.utils import is_hip
14
15 logger = init_logger(__name__)
16
@@ -319,34 +319,26 @@ def fused_topk(
319
320 M, _ = hidden_states.shape
321
322 - if is_hip():
323 - # The MoE kernels are not yet supported on ROCm.
324 - routing_weights = torch.softmax(gating_output,
325 - dim=-1,
326 - dtype=torch.float32)
327 - topk_weights, topk_ids = torch.topk(routing_weights, topk, dim=-1)
328 - else:
329 - import vllm._moe_C as moe_kernels
330 -
331 - topk_weights = torch.empty(M,
332 - topk,
333 - dtype=torch.float32,
334 - device=hidden_states.device)
335 - topk_ids = torch.empty(M,
322 + topk_weights = torch.empty(M,
323 topk,
337 - dtype=torch.int32,
324 + dtype=torch.float32,
325 device=hidden_states.device)
339 - token_expert_indicies = torch.empty(M,
340 - topk,
341 - dtype=torch.int32,
342 - device=hidden_states.device)
343 - moe_kernels.topk_softmax(
344 - topk_weights,
345 - topk_ids,
346 - token_expert_indicies,
347 - gating_output.float(), # TODO(woosuk): Optimize this.
348 - )
349 - del token_expert_indicies # Not used. Will be used in the future.
326 + topk_ids = torch.empty(M,
327 + topk,
328 + dtype=torch.int32,
329 + device=hidden_states.device)
330 + token_expert_indicies = torch.empty(M,
331 + topk,
332 + dtype=torch.int32,
333 + device=hidden_states.device)
334 + moe_kernels.topk_softmax(
335 + topk_weights,
336 + topk_ids,
337 + token_expert_indicies,
338 + gating_output.float(), # TODO(woosuk): Optimize this.
339 + )
340 + del token_expert_indicies # Not used. Will be used in the future.
341 +
342 if renormalize:
343 topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
344 return topk_weights, topk_ids