A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[ROCm] Add support for Punica kernels on AMD GPUs (#3140)
Co-authored-by: miloice <jeffaw99@hotmail.com>
kliuae committed
May 10, 2024 at 00:19 UTC
ff5abcd7463211251bcf19916ae3b45d762f48d4
10 files changed
+287
-26
CMakeLists.txt
+10
-6
@@ -219,7 +219,8 @@ set(VLLM_PUNICA_EXT_SRC
219
"csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
220
"csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
221
"csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
222
- "csrc/punica/punica_ops.cc")
222
+ "csrc/punica/punica_ops.cu"
223
+ "csrc/punica/punica_pybind.cpp")
224
225
#
226
# Copy GPU compilation flags+update for punica
@@ -243,6 +244,9 @@ if (${VLLM_GPU_LANG} STREQUAL "CUDA")
244
endif()
245
endforeach()
246
message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
247
+elseif(${VLLM_GPU_LANG} STREQUAL "HIP")
248
+ set(VLLM_PUNICA_GPU_ARCHES ${VLLM_GPU_ARCHES})
249
+ message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
250
endif()
251
252
if (VLLM_PUNICA_GPU_ARCHES)
@@ -277,11 +281,6 @@ add_custom_target(default)
281
if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
282
message(STATUS "Enabling C extension.")
283
add_dependencies(default _C)
280
-endif()
281
-
282
-if(VLLM_GPU_LANG STREQUAL "CUDA")
283
- message(STATUS "Enabling moe extension.")
284
- add_dependencies(default _moe_C)
284
285
# Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
286
# VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
@@ -292,3 +291,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
291
add_dependencies(default _punica_C)
292
endif()
293
endif()
294
+
295
+if(VLLM_GPU_LANG STREQUAL "CUDA")
296
+ message(STATUS "Enabling moe extension.")
297
+ add_dependencies(default _moe_C)
298
+endif()
Dockerfile.rocm
+3
@@ -94,6 +94,9 @@ COPY . .
94
95
RUN python3 -m pip install --upgrade pip numba
96
97
+# make sure punica kernels are built (for LoRA)
98
+ENV VLLM_INSTALL_PUNICA_KERNELS=1
99
+
100
RUN --mount=type=cache,target=/root/.cache/pip \
101
pip install -U -r requirements-rocm.txt \
102
&& patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
csrc/cuda_compat.h
+6
@@ -28,6 +28,12 @@
28
#define VLLM_SHFL_SYNC(var, src_lane) __shfl(var, src_lane)
29
#endif
30
31
+#ifndef USE_ROCM
32
+ #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down_sync(uint32_t(-1), var, lane_delta)
33
+#else
34
+ #define VLLM_SHFL_DOWN_SYNC(var, lane_delta) __shfl_down(var, lane_delta)
35
+#endif
36
+
37
#ifndef USE_ROCM
38
#define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
39
cudaFuncSetAttribute(FUNC, cudaFuncAttributeMaxDynamicSharedMemorySize, VAL)
csrc/punica/bgmv/bgmv_impl.cuh
+154
@@ -1,8 +1,14 @@
1
#pragma once
2
3
#include <ATen/cuda/CUDAContext.h>
4
+#ifndef USE_ROCM
5
#include <cooperative_groups.h>
6
+#else
7
+#include <hip/hip_cooperative_groups.h>
8
+#endif
9
+#ifndef USE_ROCM
10
#include <cuda/pipeline>
11
+#endif
12
#include <cuda_runtime.h>
13
#include <iostream>
14
#include <stdio.h>
@@ -11,6 +17,24 @@
17
18
namespace cg = cooperative_groups;
19
20
+#ifdef USE_ROCM
21
+template <size_t len>
22
+__host__ __device__
23
+inline void* memcpy_blocking(void *dst, const void *src) {
24
+ // Does not handle the case of long datatypes
25
+ char *d = reinterpret_cast<char *>(dst);
26
+ const char *s = reinterpret_cast<const char *>(src);
27
+ size_t i = 0;
28
+#pragma unroll
29
+ for (i = 0; i < len; ++i) {
30
+ d[i] = s[i];
31
+ }
32
+ return dst;
33
+}
34
+#endif
35
+
36
+#ifndef USE_ROCM
37
+
38
// nthrs = (32, 4)
39
template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
40
size_t W_copy_size, int tx, int ty, int tz, typename in_T,
@@ -141,6 +165,81 @@ bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
165
}
166
}
167
168
+#else
169
+
170
+template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
171
+ size_t W_copy_size, int tx, int ty, int tz, typename in_T,
172
+ typename out_T, typename W_T>
173
+__global__ void
174
+bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
175
+ const W_T *__restrict__ W,
176
+ const int64_t *__restrict__ indicies, int64_t y_offset,
177
+ int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
178
+ float scale) {
179
+ size_t batch_idx = blockIdx.y;
180
+ int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
181
+ if (idx < 0) {
182
+ return;
183
+ }
184
+
185
+ size_t j = blockIdx.x;
186
+ constexpr size_t tile_size = tx * ty * vec_size;
187
+ constexpr size_t num_tiles = (feat_in + tile_size - 1) / tile_size;
188
+ __shared__ float y_warpwise[ty];
189
+
190
+ float y = 0;
191
+ vec_t<in_T, vec_size> x_vec;
192
+ vec_t<W_T, vec_size> w_vec;
193
+ size_t tile_idx;
194
+
195
+#pragma unroll
196
+ for (tile_idx = 0; tile_idx < num_tiles; ++tile_idx) {
197
+ if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
198
+ x_vec.load(X + (batch_idx * feat_in) +
199
+ tile_idx * tile_size +
200
+ (threadIdx.y * tx + threadIdx.x) * vec_size);
201
+ w_vec.load(W + (idx * feat_out + j) * feat_in +
202
+ tile_idx * tile_size +
203
+ (threadIdx.y * tx + threadIdx.x) * vec_size);
204
+ }
205
+
206
+ float sum = 0.f;
207
+#pragma unroll
208
+ for (size_t i = 0; i < vec_size; ++i) {
209
+ sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
210
+ }
211
+#pragma unroll
212
+ for (size_t offset = tx / 2; offset > 0; offset /= 2) {
213
+ sum += VLLM_SHFL_DOWN_SYNC(sum, offset);
214
+ }
215
+
216
+ __syncthreads();
217
+
218
+ if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
219
+ y += sum;
220
+ }
221
+ }
222
+
223
+ if (threadIdx.x == 0) {
224
+ y_warpwise[threadIdx.y] = y;
225
+ }
226
+ __syncthreads();
227
+
228
+ float y_write = 0.f;
229
+#pragma unroll
230
+ for (size_t i = 0; i < ty; ++i) {
231
+ y_write += y_warpwise[i];
232
+ }
233
+
234
+ // write Y;
235
+ if (threadIdx.x == 0 && threadIdx.y == 0) {
236
+ size_t y_idx = batch_idx * full_y_size + y_offset + j;
237
+ Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(y_write));
238
+ }
239
+}
240
+
241
+#endif
242
+
243
// nthrs = (2, 16, 4)
244
template <int feat_in, int feat_out, size_t vec_size, int tx, int ty, int tz,
245
typename in_T, typename out_T, typename W_T>
@@ -172,7 +271,11 @@ bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
271
float sum = 0.f;
272
#pragma unroll
273
for (size_t i = 0; i < vec_size; ++i) {
274
+#ifndef USE_ROCM
275
sum += float(w_vec[i]) * float(x_vec[i]) * scale;
276
+#else
277
+ sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
278
+#endif
279
}
280
281
cg::thread_block_tile g = cg::tiled_partition<tx>(block);
@@ -183,8 +286,14 @@ bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
286
sum = g.shfl(sum, 0);
287
288
if (threadIdx.x == 0) {
289
+#ifndef USE_ROCM
290
Y[batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
291
threadIdx.z * ty + threadIdx.y] += static_cast<out_T>(sum);
292
+#else
293
+ size_t y_idx = batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
294
+ threadIdx.z * ty + threadIdx.y;
295
+ Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(sum));
296
+#endif
297
}
298
}
299
@@ -236,6 +345,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
345
scale);
346
}
347
} else {
348
+#ifndef USE_ROCM
349
static_assert(feat_in % (vec_size * 32) == 0 ||
350
feat_in % (vec_size * 16) == 0 ||
351
feat_in % (vec_size * 8) == 0);
@@ -279,6 +389,50 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
389
full_y_size, num_layers, layer_idx,
390
scale);
391
}
392
+#else
393
+ constexpr size_t rocm_warp_size = warpSize;
394
+
395
+#define CHECK_INPUT_TILEABLE_BY(vec_size_) \
396
+ feat_in % (rocm_warp_size * vec_size_) == 0
397
+
398
+#define LAUNCH_BGMV_SHRINK_KERNELS_ROCM(factor_, vec_size_, tx_, ty_) \
399
+ if constexpr (CHECK_INPUT_TILEABLE_BY(factor_)) { \
400
+ constexpr size_t vec_size_shrink = vec_size_; \
401
+ constexpr int tx = tx_; \
402
+ constexpr int ty = ty_; \
403
+ dim3 nblks(feat_out, batch_size); \
404
+ dim3 nthrs(tx, ty); \
405
+ bgmv_shrink_kernel<feat_in, feat_out, vec_size_shrink, \
406
+ vec_size_shrink * sizeof(in_T), \
407
+ vec_size_shrink * sizeof(W_T), \
408
+ tx, ty, tz> \
409
+ <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset, \
410
+ full_y_size, num_layers, layer_idx, \
411
+ scale); \
412
+ }
413
+
414
+ static_assert(CHECK_INPUT_TILEABLE_BY(32) ||
415
+ CHECK_INPUT_TILEABLE_BY(16) ||
416
+ CHECK_INPUT_TILEABLE_BY( 8) ||
417
+ CHECK_INPUT_TILEABLE_BY( 4) ||
418
+ CHECK_INPUT_TILEABLE_BY( 2) ||
419
+ CHECK_INPUT_TILEABLE_BY( 1));
420
+
421
+ LAUNCH_BGMV_SHRINK_KERNELS_ROCM(32, vec_size, rocm_warp_size, 32/vec_size)
422
+ else
423
+ LAUNCH_BGMV_SHRINK_KERNELS_ROCM(16, vec_size, rocm_warp_size, 16/vec_size)
424
+ else
425
+ LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 8, vec_size, rocm_warp_size, 8/vec_size)
426
+ else
427
+ LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 4, vec_size, rocm_warp_size/(vec_size/4), vec_size/4)
428
+ else
429
+ LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 2, vec_size, rocm_warp_size/(vec_size/2), vec_size/2)
430
+ else
431
+ LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 1, vec_size, rocm_warp_size/(vec_size/1), vec_size/1)
432
+
433
+#undef CHECK_INPUT_TILEABLE_BY
434
+#undef LAUNCH_BGMV_SHRINK_KERNELS_ROCM
435
+#endif
436
}
437
}
438
csrc/punica/bgmv/vec_dtypes.cuh
+3
-2
@@ -1,8 +1,6 @@
1
#ifndef VEC_DTYPES_CUH_
2
#define VEC_DTYPES_CUH_
3
4
-#include <cuda_bf16.h>
5
-#include <cuda_fp16.h>
4
#ifdef FLASHINFER_USE_FP8
5
#include <cuda_fp8.h>
6
#endif
@@ -10,6 +8,9 @@
8
9
#include <type_traits>
10
11
+#include "../type_convert.h"
12
+#include "../../cuda_compat.h"
13
+
14
#define FLASHINFER_INLINE \
15
inline __attribute__((always_inline)) __device__ __host__
16
csrc/punica/punica_ops.cu
renamed
+2
-15
@@ -1,12 +1,11 @@
1
-#include <cuda_bf16.h>
2
-#include <cuda_fp16.h>
1
#include <torch/extension.h>
2
#include <c10/cuda/CUDAGuard.h>
3
#include <cstdint>
4
5
+#include "type_convert.h"
6
+#include "../cuda_compat.h"
7
#include "bgmv/bgmv_config.h"
8
9
-namespace {
9
10
//====== utils ======
11
@@ -568,15 +567,3 @@ void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
567
TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
568
" dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
569
}
571
-
572
-} // namespace
573
-
574
-//====== pybind ======
575
-
576
-#define DEFINE_pybind(name) m.def(#name, &name, #name);
577
-
578
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
579
- m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
580
- m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
581
- "dispatch_bgmv_low_level");
582
-}
csrc/punica/punica_ops.h
new
+11
@@ -0,0 +1,11 @@
1
+#pragma once
2
+
3
+#include <torch/extension.h>
4
+
5
+void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
6
+ torch::Tensor indicies, int64_t layer_idx, float scale);
7
+
8
+void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
9
+ torch::Tensor indicies, int64_t layer_idx,
10
+ float scale, int64_t h_in, int64_t h_out,
11
+ int64_t y_offset);
csrc/punica/punica_pybind.cpp
new
+13
@@ -0,0 +1,13 @@
1
+#include <torch/extension.h>
2
+
3
+#include "punica_ops.h"
4
+
5
+//====== pybind ======
6
+
7
+#define DEFINE_pybind(name) m.def(#name, &name, #name);
8
+
9
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
10
+ m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
11
+ m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
12
+ "dispatch_bgmv_low_level");
13
+}
csrc/punica/type_convert.h
new
+82
@@ -0,0 +1,82 @@
1
+#ifndef CSRC__PUNICA__TYPE_CONVERT_H__
2
+#define CSRC__PUNICA__TYPE_CONVERT_H__
3
+
4
+#ifndef USE_ROCM
5
+
6
+#include <cuda_bf16.h>
7
+#include <cuda_fp16.h>
8
+
9
+#else
10
+
11
+#include <hip/hip_bf16.h>
12
+#include <hip/hip_fp16.h>
13
+
14
+#define __TYPE_CONVERT__HOST_DEVICE__ __host__ __device__
15
+
16
+typedef __half nv_half;
17
+typedef __hip_bfloat16 nv_bfloat16;
18
+typedef __hip_bfloat162 nv_bfloat162;
19
+
20
+__TYPE_CONVERT__HOST_DEVICE__
21
+inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 val) {
22
+ return __hip_bfloat162{val, val};
23
+}
24
+
25
+__TYPE_CONVERT__HOST_DEVICE__
26
+inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 vall, __hip_bfloat16 valr) {
27
+ return __hip_bfloat162{vall, valr};
28
+}
29
+
30
+template <typename T_src, typename T_dst>
31
+__TYPE_CONVERT__HOST_DEVICE__
32
+inline T_dst convert_type(T_src val) {
33
+ return static_cast<T_dst>(val);
34
+}
35
+
36
+template <>
37
+__TYPE_CONVERT__HOST_DEVICE__
38
+inline float convert_type<__half, float>(__half val) {
39
+ return __half2float(val);
40
+}
41
+
42
+template <>
43
+__TYPE_CONVERT__HOST_DEVICE__
44
+inline __half convert_type<float, __half>(float val) {
45
+ return __float2half(val);
46
+}
47
+
48
+template <>
49
+__TYPE_CONVERT__HOST_DEVICE__
50
+inline float convert_type<__hip_bfloat16, float>(__hip_bfloat16 val) {
51
+ return __bfloat162float(val);
52
+}
53
+
54
+template <>
55
+__TYPE_CONVERT__HOST_DEVICE__
56
+inline __hip_bfloat16 convert_type<float, __hip_bfloat16>(float val) {
57
+ return __float2bfloat16(val);
58
+}
59
+
60
+template <typename T>
61
+__TYPE_CONVERT__HOST_DEVICE__
62
+inline T vllm_add(T a, T b) {
63
+ return a + b;
64
+}
65
+
66
+template <>
67
+__TYPE_CONVERT__HOST_DEVICE__
68
+inline __half vllm_add<__half>(__half a, __half b) {
69
+ return __hadd(a, b);
70
+}
71
+
72
+template <>
73
+__TYPE_CONVERT__HOST_DEVICE__
74
+inline __hip_bfloat16 vllm_add<__hip_bfloat16>(__hip_bfloat16 a, __hip_bfloat16 b) {
75
+ return __hadd(a, b);
76
+}
77
+
78
+#undef __TYPE_CONVERT__HOST_DEVICE__
79
+
80
+#endif // USE_ROCM
81
+
82
+#endif // CSRC__PUNICA__TYPE_CONVERT_H__
setup.py
+3
-3
@@ -385,12 +385,12 @@ ext_modules = []
385
if _is_cuda():
386
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
387
388
- if _install_punica():
389
- ext_modules.append(CMakeExtension(name="vllm._punica_C"))
390
-
388
if not _is_neuron():
389
ext_modules.append(CMakeExtension(name="vllm._C"))
390
391
+ if _install_punica():
392
+ ext_modules.append(CMakeExtension(name="vllm._punica_C"))
393
+
394
package_data = {
395
"vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
396
}