@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ff5abcd746
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[ROCm] Add support for Punica kernels on AMD GPUs (#3140)

Co-authored-by: miloice <jeffaw99@hotmail.com>

kliuae committed May 10, 2024 at 00:19 UTC ff5abcd7463211251bcf19916ae3b45d762f48d4
10 files changed +287 -26
CMakeLists.txt
+10 -6
@@ -219,7 +219,8 @@ set(VLLM_PUNICA_EXT_SRC
219 "csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
220 "csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
221 "csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
222 - "csrc/punica/punica_ops.cc")
222 + "csrc/punica/punica_ops.cu"
223 + "csrc/punica/punica_pybind.cpp")
224
225 #
226 # Copy GPU compilation flags+update for punica
@@ -243,6 +244,9 @@ if (${VLLM_GPU_LANG} STREQUAL "CUDA")
244 endif()
245 endforeach()
246 message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
247 +elseif(${VLLM_GPU_LANG} STREQUAL "HIP")
248 + set(VLLM_PUNICA_GPU_ARCHES ${VLLM_GPU_ARCHES})
249 + message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
250 endif()
251
252 if (VLLM_PUNICA_GPU_ARCHES)
@@ -277,11 +281,6 @@ add_custom_target(default)
281 if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
282 message(STATUS "Enabling C extension.")
283 add_dependencies(default _C)
280 -endif()
281 -
282 -if(VLLM_GPU_LANG STREQUAL "CUDA")
283 - message(STATUS "Enabling moe extension.")
284 - add_dependencies(default _moe_C)
284
285 # Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
286 # VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
@@ -292,3 +291,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
291 add_dependencies(default _punica_C)
292 endif()
293 endif()
294 +
295 +if(VLLM_GPU_LANG STREQUAL "CUDA")
296 + message(STATUS "Enabling moe extension.")
297 + add_dependencies(default _moe_C)
298 +endif()
Dockerfile.rocm
+3
@@ -94,6 +94,9 @@ COPY . .
94
95 RUN python3 -m pip install --upgrade pip numba
96
97 +# make sure punica kernels are built (for LoRA)
98 +ENV VLLM_INSTALL_PUNICA_KERNELS=1
99 +
100 RUN --mount=type=cache,target=/root/.cache/pip \
101 pip install -U -r requirements-rocm.txt \
102 && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
csrc/cuda_compat.h
+6
@@ -28,6 +28,12 @@
28 #define VLLM_SHFL_SYNC(var, src_lane) __shfl(var, src_lane)
29 #endif
30
31 +#ifndef USE_ROCM
32 + #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down_sync(uint32_t(-1), var, lane_delta)
33 +#else
34 + #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down(var, lane_delta)
35 +#endif
36 +
37 #ifndef USE_ROCM
38 #define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
39 cudaFuncSetAttribute(FUNC, cudaFuncAttributeMaxDynamicSharedMemorySize, VAL)
csrc/punica/bgmv/bgmv_impl.cuh
+154
@@ -1,8 +1,14 @@
1 #pragma once
2
3 #include <ATen/cuda/CUDAContext.h>
4 +#ifndef USE_ROCM
5 #include <cooperative_groups.h>
6 +#else
7 +#include <hip/hip_cooperative_groups.h>
8 +#endif
9 +#ifndef USE_ROCM
10 #include <cuda/pipeline>
11 +#endif
12 #include <cuda_runtime.h>
13 #include <iostream>
14 #include <stdio.h>
@@ -11,6 +17,24 @@
17
18 namespace cg = cooperative_groups;
19
20 +#ifdef USE_ROCM
21 +template <size_t len>
22 +__host__ __device__
23 +inline void* memcpy_blocking(void *dst, const void *src) {
24 + // Does not handle the case of long datatypes
25 + char *d = reinterpret_cast<char *>(dst);
26 + const char *s = reinterpret_cast<const char *>(src);
27 + size_t i = 0;
28 +#pragma unroll
29 + for (i = 0; i < len; ++i) {
30 + d[i] = s[i];
31 + }
32 + return dst;
33 +}
34 +#endif
35 +
36 +#ifndef USE_ROCM
37 +
38 // nthrs = (32, 4)
39 template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
40 size_t W_copy_size, int tx, int ty, int tz, typename in_T,
@@ -141,6 +165,81 @@ bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
165 }
166 }
167
168 +#else
169 +
170 +template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
171 + size_t W_copy_size, int tx, int ty, int tz, typename in_T,
172 + typename out_T, typename W_T>
173 +__global__ void
174 +bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
175 + const W_T *__restrict__ W,
176 + const int64_t *__restrict__ indicies, int64_t y_offset,
177 + int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
178 + float scale) {
179 + size_t batch_idx = blockIdx.y;
180 + int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
181 + if (idx < 0) {
182 + return;
183 + }
184 +
185 + size_t j = blockIdx.x;
186 + constexpr size_t tile_size = tx * ty * vec_size;
187 + constexpr size_t num_tiles = (feat_in + tile_size - 1) / tile_size;
188 + __shared__ float y_warpwise[ty];
189 +
190 + float y = 0;
191 + vec_t<in_T, vec_size> x_vec;
192 + vec_t<W_T, vec_size> w_vec;
193 + size_t tile_idx;
194 +
195 +#pragma unroll
196 + for (tile_idx = 0; tile_idx < num_tiles; ++tile_idx) {
197 + if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
198 + x_vec.load(X + (batch_idx * feat_in) +
199 + tile_idx * tile_size +
200 + (threadIdx.y * tx + threadIdx.x) * vec_size);
201 + w_vec.load(W + (idx * feat_out + j) * feat_in +
202 + tile_idx * tile_size +
203 + (threadIdx.y * tx + threadIdx.x) * vec_size);
204 + }
205 +
206 + float sum = 0.f;
207 +#pragma unroll
208 + for (size_t i = 0; i < vec_size; ++i) {
209 + sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
210 + }
211 +#pragma unroll
212 + for (size_t offset = tx / 2; offset > 0; offset /= 2) {
213 + sum += VLLM_SHFL_DOWN_SYNC(sum, offset);
214 + }
215 +
216 + __syncthreads();
217 +
218 + if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
219 + y += sum;
220 + }
221 + }
222 +
223 + if (threadIdx.x == 0) {
224 + y_warpwise[threadIdx.y] = y;
225 + }
226 + __syncthreads();
227 +
228 + float y_write = 0.f;
229 +#pragma unroll
230 + for (size_t i = 0; i < ty; ++i) {
231 + y_write += y_warpwise[i];
232 + }
233 +
234 + // write Y;
235 + if (threadIdx.x == 0 && threadIdx.y == 0) {
236 + size_t y_idx = batch_idx * full_y_size + y_offset + j;
237 + Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(y_write));
238 + }
239 +}
240 +
241 +#endif
242 +
243 // nthrs = (2, 16, 4)
244 template <int feat_in, int feat_out, size_t vec_size, int tx, int ty, int tz,
245 typename in_T, typename out_T, typename W_T>
@@ -172,7 +271,11 @@ bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
271 float sum = 0.f;
272 #pragma unroll
273 for (size_t i = 0; i < vec_size; ++i) {
274 +#ifndef USE_ROCM
275 sum += float(w_vec[i]) * float(x_vec[i]) * scale;
276 +#else
277 + sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
278 +#endif
279 }
280
281 cg::thread_block_tile g = cg::tiled_partition<tx>(block);
@@ -183,8 +286,14 @@ bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
286 sum = g.shfl(sum, 0);
287
288 if (threadIdx.x == 0) {
289 +#ifndef USE_ROCM
290 Y[batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
291 threadIdx.z * ty + threadIdx.y] += static_cast<out_T>(sum);
292 +#else
293 + size_t y_idx = batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
294 + threadIdx.z * ty + threadIdx.y;
295 + Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(sum));
296 +#endif
297 }
298 }
299
@@ -236,6 +345,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
345 scale);
346 }
347 } else {
348 +#ifndef USE_ROCM
349 static_assert(feat_in % (vec_size * 32) == 0 ||
350 feat_in % (vec_size * 16) == 0 ||
351 feat_in % (vec_size * 8) == 0);
@@ -279,6 +389,50 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
389 full_y_size, num_layers, layer_idx,
390 scale);
391 }
392 +#else
393 + constexpr size_t rocm_warp_size = warpSize;
394 +
395 +#define CHECK_INPUT_TILEABLE_BY(vec_size_) \
396 + feat_in % (rocm_warp_size * vec_size_) == 0
397 +
398 +#define LAUNCH_BGMV_SHRINK_KERNELS_ROCM(factor_, vec_size_, tx_, ty_) \
399 + if constexpr (CHECK_INPUT_TILEABLE_BY(factor_)) { \
400 + constexpr size_t vec_size_shrink = vec_size_; \
401 + constexpr int tx = tx_; \
402 + constexpr int ty = ty_; \
403 + dim3 nblks(feat_out, batch_size); \
404 + dim3 nthrs(tx, ty); \
405 + bgmv_shrink_kernel<feat_in, feat_out, vec_size_shrink, \
406 + vec_size_shrink * sizeof(in_T), \
407 + vec_size_shrink * sizeof(W_T), \
408 + tx, ty, tz> \
409 + <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset, \
410 + full_y_size, num_layers, layer_idx, \
411 + scale); \
412 + }
413 +
414 + static_assert(CHECK_INPUT_TILEABLE_BY(32) ||
415 + CHECK_INPUT_TILEABLE_BY(16) ||
416 + CHECK_INPUT_TILEABLE_BY( 8) ||
417 + CHECK_INPUT_TILEABLE_BY( 4) ||
418 + CHECK_INPUT_TILEABLE_BY( 2) ||
419 + CHECK_INPUT_TILEABLE_BY( 1));
420 +
421 + LAUNCH_BGMV_SHRINK_KERNELS_ROCM(32, vec_size, rocm_warp_size, 32/vec_size)
422 + else
423 + LAUNCH_BGMV_SHRINK_KERNELS_ROCM(16, vec_size, rocm_warp_size, 16/vec_size)
424 + else
425 + LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 8, vec_size, rocm_warp_size, 8/vec_size)
426 + else
427 + LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 4, vec_size, rocm_warp_size/(vec_size/4), vec_size/4)
428 + else
429 + LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 2, vec_size, rocm_warp_size/(vec_size/2), vec_size/2)
430 + else
431 + LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 1, vec_size, rocm_warp_size/(vec_size/1), vec_size/1)
432 +
433 +#undef CHECK_INPUT_TILEABLE_BY
434 +#undef LAUNCH_BGMV_SHRINK_KERNELS_ROCM
435 +#endif
436 }
437 }
438
csrc/punica/bgmv/vec_dtypes.cuh
+3 -2
@@ -1,8 +1,6 @@
1 #ifndef VEC_DTYPES_CUH_
2 #define VEC_DTYPES_CUH_
3
4 -#include <cuda_bf16.h>
5 -#include <cuda_fp16.h>
4 #ifdef FLASHINFER_USE_FP8
5 #include <cuda_fp8.h>
6 #endif
@@ -10,6 +8,9 @@
8
9 #include <type_traits>
10
11 +#include "../type_convert.h"
12 +#include "../../cuda_compat.h"
13 +
14 #define FLASHINFER_INLINE \
15 inline __attribute__((always_inline)) __device__ __host__
16
csrc/punica/punica_ops.cu renamed
+2 -15
@@ -1,12 +1,11 @@
1 -#include <cuda_bf16.h>
2 -#include <cuda_fp16.h>
1 #include <torch/extension.h>
2 #include <c10/cuda/CUDAGuard.h>
3 #include <cstdint>
4
5 +#include "type_convert.h"
6 +#include "../cuda_compat.h"
7 #include "bgmv/bgmv_config.h"
8
9 -namespace {
9
10 //====== utils ======
11
@@ -568,15 +567,3 @@ void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
567 TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
568 " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
569 }
571 -
572 -} // namespace
573 -
574 -//====== pybind ======
575 -
576 -#define DEFINE_pybind(name) m.def(#name, &name, #name);
577 -
578 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
579 - m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
580 - m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
581 - "dispatch_bgmv_low_level");
582 -}
csrc/punica/punica_ops.h new
+11
@@ -0,0 +1,11 @@
1 +#pragma once
2 +
3 +#include <torch/extension.h>
4 +
5 +void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
6 + torch::Tensor indicies, int64_t layer_idx, float scale);
7 +
8 +void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
9 + torch::Tensor indicies, int64_t layer_idx,
10 + float scale, int64_t h_in, int64_t h_out,
11 + int64_t y_offset);
csrc/punica/punica_pybind.cpp new
+13
@@ -0,0 +1,13 @@
1 +#include <torch/extension.h>
2 +
3 +#include "punica_ops.h"
4 +
5 +//====== pybind ======
6 +
7 +#define DEFINE_pybind(name) m.def(#name, &name, #name);
8 +
9 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
10 + m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
11 + m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
12 + "dispatch_bgmv_low_level");
13 +}
csrc/punica/type_convert.h new
+82
@@ -0,0 +1,82 @@
1 +#ifndef CSRC__PUNICA__TYPE_CONVERT_H__
2 +#define CSRC__PUNICA__TYPE_CONVERT_H__
3 +
4 +#ifndef USE_ROCM
5 +
6 +#include <cuda_bf16.h>
7 +#include <cuda_fp16.h>
8 +
9 +#else
10 +
11 +#include <hip/hip_bf16.h>
12 +#include <hip/hip_fp16.h>
13 +
14 +#define __TYPE_CONVERT__HOST_DEVICE__ __host__ __device__
15 +
16 +typedef __half nv_half;
17 +typedef __hip_bfloat16 nv_bfloat16;
18 +typedef __hip_bfloat162 nv_bfloat162;
19 +
20 +__TYPE_CONVERT__HOST_DEVICE__
21 +inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 val) {
22 + return __hip_bfloat162{val, val};
23 +}
24 +
25 +__TYPE_CONVERT__HOST_DEVICE__
26 +inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 vall, __hip_bfloat16 valr) {
27 + return __hip_bfloat162{vall, valr};
28 +}
29 +
30 +template <typename T_src, typename T_dst>
31 +__TYPE_CONVERT__HOST_DEVICE__
32 +inline T_dst convert_type(T_src val) {
33 + return static_cast<T_dst>(val);
34 +}
35 +
36 +template <>
37 +__TYPE_CONVERT__HOST_DEVICE__
38 +inline float convert_type<__half, float>(__half val) {
39 + return __half2float(val);
40 +}
41 +
42 +template <>
43 +__TYPE_CONVERT__HOST_DEVICE__
44 +inline __half convert_type<float, __half>(float val) {
45 + return __float2half(val);
46 +}
47 +
48 +template <>
49 +__TYPE_CONVERT__HOST_DEVICE__
50 +inline float convert_type<__hip_bfloat16, float>(__hip_bfloat16 val) {
51 + return __bfloat162float(val);
52 +}
53 +
54 +template <>
55 +__TYPE_CONVERT__HOST_DEVICE__
56 +inline __hip_bfloat16 convert_type<float, __hip_bfloat16>(float val) {
57 + return __float2bfloat16(val);
58 +}
59 +
60 +template <typename T>
61 +__TYPE_CONVERT__HOST_DEVICE__
62 +inline T vllm_add(T a, T b) {
63 + return a + b;
64 +}
65 +
66 +template <>
67 +__TYPE_CONVERT__HOST_DEVICE__
68 +inline __half vllm_add<__half>(__half a, __half b) {
69 + return __hadd(a, b);
70 +}
71 +
72 +template <>
73 +__TYPE_CONVERT__HOST_DEVICE__
74 +inline __hip_bfloat16 vllm_add<__hip_bfloat16>(__hip_bfloat16 a, __hip_bfloat16 b) {
75 + return __hadd(a, b);
76 +}
77 +
78 +#undef __TYPE_CONVERT__HOST_DEVICE__
79 +
80 +#endif // USE_ROCM
81 +
82 +#endif // CSRC__PUNICA__TYPE_CONVERT_H__
setup.py
+3 -3
@@ -385,12 +385,12 @@ ext_modules = []
385 if _is_cuda():
386 ext_modules.append(CMakeExtension(name="vllm._moe_C"))
387
388 - if _install_punica():
389 - ext_modules.append(CMakeExtension(name="vllm._punica_C"))
390 -
388 if not _is_neuron():
389 ext_modules.append(CMakeExtension(name="vllm._C"))
390
391 + if _install_punica():
392 + ext_modules.append(CMakeExtension(name="vllm._punica_C"))
393 +
394 package_data = {
395 "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
396 }