A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Merge EmbeddedLLM/vllm-rocm into vLLM main (#1836)
Co-authored-by: Philipp Moritz <pcmoritz@gmail.com> Co-authored-by: Amir Balwel <amoooori04@gmail.com> Co-authored-by: root <kuanfu.liu@akirakan.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: kuanfu <kuanfu.liu@embeddedllm.com> Co-authored-by: miloice <17350011+kliuae@users.noreply.github.com>
TJian committed
Dec 8, 2023 at 15:16 UTC
6ccc0bfffbcf1b7e927cc3dcf4159fc74ff94d40
29 files changed
+873
-118
.gitignore
+4
@@ -177,3 +177,7 @@ _build/
177
# vim swap files
178
*.swo
179
*.swp
180
+
181
+# hip files generated by PyTorch
182
+*.hip
183
+*_hip*
Dockerfile.rocm
new
+62
@@ -0,0 +1,62 @@
1
+FROM rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1
2
+
3
+# Install some basic utilities
4
+RUN apt-get update && apt-get install python3 python3-pip -y
5
+
6
+# Install some basic utilities
7
+RUN apt-get update && apt-get install -y \
8
+ curl \
9
+ ca-certificates \
10
+ sudo \
11
+ git \
12
+ bzip2 \
13
+ libx11-6 \
14
+ build-essential \
15
+ wget \
16
+ unzip \
17
+ nvidia-cuda-toolkit \
18
+ tmux \
19
+ && rm -rf /var/lib/apt/lists/*
20
+
21
+### Mount Point ###
22
+# When launching the container, mount the code directory to /app
23
+ARG APP_MOUNT=/app
24
+VOLUME [ ${APP_MOUNT} ]
25
+WORKDIR ${APP_MOUNT}
26
+
27
+RUN python3 -m pip install --upgrade pip
28
+RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
29
+
30
+ENV LLVM_SYMBOLIZER_PATH=/opt/rocm/llvm/bin/llvm-symbolizer
31
+ENV PATH=$PATH:/opt/rocm/bin:/libtorch/bin:
32
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
33
+ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
34
+
35
+# Install ROCm flash-attention
36
+RUN mkdir libs \
37
+ && cd libs \
38
+ && git clone https://github.com/ROCmSoftwarePlatform/flash-attention.git \
39
+ && cd flash-attention \
40
+ && git checkout 3d2b6f5 \
41
+ && git submodule update --init \
42
+ && export GPU_ARCHS=$(/opt/rocm/llvm/bin/amdgpu-offload-arch) \
43
+ && patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch \
44
+ && python3 setup.py install \
45
+ && cd ..
46
+
47
+COPY ./ /app/vllm
48
+
49
+RUN python3 -m pip install --upgrade pip
50
+RUN pip install xformers==0.0.22.post7 --no-deps
51
+
52
+RUN cd /app \
53
+ && cd vllm \
54
+ && pip install -U -r requirements-rocm.txt \
55
+ && bash patch_xformers-0.0.22.post7.rocm.sh \
56
+ && python3 setup.py install \
57
+ && cd ..
58
+
59
+RUN python3 -m pip install --upgrade pip
60
+RUN python3 -m pip install --no-cache-dir ray[all]
61
+
62
+CMD ["/bin/bash"]
README.md
+2
@@ -17,6 +17,7 @@ Easy, fast, and cheap LLM serving for everyone
17
---
18
19
*Latest News* 🔥
20
+- [2023/12] Added ROCm support to vLLM.
21
- [2023/10] We hosted [the first vLLM meetup](https://lu.ma/first-vllm-meetup) in SF! Please find the meetup slides [here](https://docs.google.com/presentation/d/1QL-XPFXiFpDBh86DbEegFXBXFXjix4v032GhShbKf3s/edit?usp=sharing).
22
- [2023/09] We created our [Discord server](https://discord.gg/jz7wjKhh6g)! Join us to discuss vLLM and LLM serving! We will also post the latest announcements and updates there.
23
- [2023/09] We released our [PagedAttention paper](https://arxiv.org/abs/2309.06180) on arXiv!
@@ -43,6 +44,7 @@ vLLM is flexible and easy to use with:
44
- Tensor parallelism support for distributed inference
45
- Streaming outputs
46
- OpenAI-compatible API server
47
+- Support NVIDIA CUDA and AMD ROCm.
48
49
vLLM seamlessly supports many Hugging Face models, including the following architectures:
50
csrc/activation_kernels.cu
+4
-3
@@ -1,6 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
+#include "cuda_compat.h"
5
#include "dispatch_utils.h"
6
7
namespace vllm {
@@ -18,8 +19,8 @@ __global__ void silu_and_mul_kernel(
19
const int d) {
20
const int64_t token_idx = blockIdx.x;
21
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
21
- const scalar_t x = __ldg(&input[token_idx * 2 * d + idx]);
22
- const scalar_t y = __ldg(&input[token_idx * 2 * d + d + idx]);
22
+ const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
23
+ const scalar_t y = VLLM_LDG(&input[token_idx * 2 * d + d + idx]);
24
out[token_idx * d + idx] = silu(x) * y;
25
}
26
}
@@ -57,7 +58,7 @@ __global__ void activation_kernel(
58
const int d) {
59
const int64_t token_idx = blockIdx.x;
60
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
60
- const scalar_t x = __ldg(&input[token_idx * d + idx]);
61
+ const scalar_t x = VLLM_LDG(&input[token_idx * d + idx]);
62
out[token_idx * d + idx] = ACT_FN(x);
63
}
64
}
csrc/attention/attention_kernels.cu
+21
-13
@@ -15,6 +15,10 @@
15
* See the License for the specific language governing permissions and
16
* limitations under the License.
17
*/
18
+#ifdef USE_ROCM
19
+#include <hip/hip_runtime.h>
20
+#endif
21
+
22
#include <torch/extension.h>
23
#include <ATen/cuda/CUDAContext.h>
24
@@ -23,7 +27,11 @@
27
28
#include <algorithm>
29
30
+#ifndef USE_ROCM
31
#define WARP_SIZE 32
32
+#else
33
+#define WARP_SIZE warpSize
34
+#endif
35
#define MAX(a, b) ((a) > (b) ? (a) : (b))
36
#define MIN(a, b) ((a) < (b) ? (a) : (b))
37
#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
@@ -40,7 +48,7 @@ inline __device__ float block_sum(float* red_smem, float sum) {
48
// Compute the sum per warp.
49
#pragma unroll
50
for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
43
- sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
51
+ sum += VLLM_SHFL_XOR_SYNC(sum, mask);
52
}
53
54
// Warp leaders store the data to shared memory.
@@ -59,11 +67,11 @@ inline __device__ float block_sum(float* red_smem, float sum) {
67
// Parallel reduction inside the warp.
68
#pragma unroll
69
for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
62
- sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
70
+ sum += VLLM_SHFL_XOR_SYNC(sum, mask);
71
}
72
73
// Broadcast to other threads.
66
- return __shfl_sync(uint32_t(-1), sum, 0);
74
+ return VLLM_SHFL_SYNC(sum, 0);
75
}
76
77
// TODO(woosuk): Merge the last two dimensions of the grid.
@@ -223,7 +231,7 @@ __device__ void paged_attention_kernel(
231
// The 0-th thread of each thread group already has its max qk value.
232
#pragma unroll
233
for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
226
- qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
234
+ qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
235
}
236
if (lane == 0) {
237
red_smem[warp_idx] = qk_max;
@@ -235,10 +243,10 @@ __device__ void paged_attention_kernel(
243
qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
244
#pragma unroll
245
for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
238
- qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
246
+ qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
247
}
248
// Broadcast the max qk value to all threads.
241
- qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
249
+ qk_max = VLLM_SHFL_SYNC(qk_max, 0);
250
251
// Get the sum of the exp values.
252
float exp_sum = 0.f;
@@ -326,7 +334,7 @@ __device__ void paged_attention_kernel(
334
float acc = accs[i];
335
#pragma unroll
336
for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
329
- acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
337
+ acc += VLLM_SHFL_XOR_SYNC(acc, mask);
338
}
339
accs[i] = acc;
340
}
@@ -492,7 +500,7 @@ __global__ void paged_attention_v2_reduce_kernel(
500
// Reduce within the warp.
501
#pragma unroll
502
for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
495
- max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
503
+ max_logit = fmaxf(max_logit, VLLM_SHFL_XOR_SYNC(max_logit, mask));
504
}
505
if (lane == 0) {
506
red_smem[warp_idx] = max_logit;
@@ -502,10 +510,10 @@ __global__ void paged_attention_v2_reduce_kernel(
510
max_logit = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
511
#pragma unroll
512
for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
505
- max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
513
+ max_logit = fmaxf(max_logit, VLLM_SHFL_XOR_SYNC(max_logit, mask));
514
}
515
// Broadcast the max value to all threads.
508
- max_logit = __shfl_sync(uint32_t(-1), max_logit, 0);
516
+ max_logit = VLLM_SHFL_SYNC(max_logit, 0);
517
518
// Load rescaled exp sums to shared memory.
519
float* shared_exp_sums = reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
@@ -539,9 +547,9 @@ __global__ void paged_attention_v2_reduce_kernel(
547
} // namespace vllm
548
549
#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
542
- cudaFuncSetAttribute( \
543
- vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>, \
544
- cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem_size); \
550
+ VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
551
+ ((void*)vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>), \
552
+ shared_mem_size); \
553
vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
554
<<<grid, block, shared_mem_size, stream>>>( \
555
out_ptr, \
csrc/attention/attention_utils.cuh
+2
-1
@@ -17,6 +17,7 @@
17
*/
18
#pragma once
19
20
+#include "../cuda_compat.h"
21
#include "attention_dtypes.h"
22
23
#include <float.h>
@@ -39,7 +40,7 @@ inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
40
float qk = sum(qk_vec);
41
#pragma unroll
42
for (int mask = THREAD_GROUP_SIZE / 2; mask >= 1; mask /= 2) {
42
- qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
43
+ qk += VLLM_SHFL_XOR_SYNC(qk, mask);
44
}
45
return qk;
46
}
csrc/attention/dtype_bfloat16.cuh
+16
-3
@@ -21,8 +21,17 @@
21
#include "attention_generic.cuh"
22
#include "dtype_float32.cuh"
23
24
-#include <cuda_bf16.h>
25
-#include <cuda_fp16.h>
24
+#ifndef USE_ROCM
25
+ #include <cuda_bf16.h>
26
+ #include <cuda_fp16.h>
27
+#else
28
+ #include <hip/hip_bf16.h>
29
+ #include <hip/hip_fp16.h>
30
+
31
+ typedef __hip_bfloat162 __nv_bfloat162;
32
+ typedef __hip_bfloat16 __nv_bfloat16;
33
+#endif
34
+
35
#include <stdint.h>
36
37
namespace vllm {
@@ -98,7 +107,11 @@ inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
107
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
108
assert(false);
109
#else
101
- return a + b;
110
+ #ifndef USE_ROCM
111
+ return a + b;
112
+ #else
113
+ return __hadd(a, b);
114
+ #endif
115
#endif
116
}
117
csrc/attention/dtype_float16.cuh
+63
-5
@@ -21,6 +21,10 @@
21
#include "attention_generic.cuh"
22
#include "dtype_float32.cuh"
23
24
+#ifdef USE_ROCM
25
+ #include <hip/hip_fp16.h>
26
+#endif
27
+
28
#include <stdint.h>
29
30
namespace vllm {
@@ -63,21 +67,47 @@ struct FloatVec<uint4> {
67
68
// Utility functions for type conversions.
69
inline __device__ uint32_t h0_h0(uint16_t a) {
70
+#ifndef USE_ROCM
71
uint32_t b;
72
asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
73
return b;
74
+#else
75
+ union {
76
+ uint32_t u32;
77
+ uint16_t u16[2];
78
+ } tmp;
79
+ tmp.u16[0] = a;
80
+ tmp.u16[1] = a;
81
+ return tmp.u32;
82
+#endif
83
}
84
85
inline __device__ float half_to_float(uint16_t h) {
86
float f;
87
+#ifndef USE_ROCM
88
asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
89
+#else
90
+ asm volatile("v_cvt_f32_f16 %0, %1;" : "=v"(f) : "v"(h));
91
+#endif
92
return f;
93
}
94
95
inline __device__ float2 half2_to_float2(uint32_t v) {
96
+#ifndef USE_ROCM
97
uint16_t lo, hi;
98
asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
99
return make_float2(half_to_float(lo), half_to_float(hi));
100
+#else
101
+ union {
102
+ uint32_t u32;
103
+ uint16_t u16[2];
104
+ } tmp;
105
+ tmp.u32 = v;
106
+ float2 ret;
107
+ ret.x = half_to_float(tmp.u16[0]);
108
+ ret.y = half_to_float(tmp.u16[1]);
109
+ return ret;
110
+#endif
111
}
112
113
inline __device__ uint16_t float_to_half(float f) {
@@ -85,7 +115,11 @@ inline __device__ uint16_t float_to_half(float f) {
115
uint32_t u32;
116
uint16_t u16[2];
117
} tmp;
118
+#ifndef USE_ROCM
119
asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
120
+#else
121
+ asm volatile("v_cvt_f16_f32 %0, %1;\n" : "=v"(tmp.u32) : "v"(f));
122
+#endif
123
return tmp.u16[0];
124
}
125
@@ -94,12 +128,16 @@ inline __device__ uint32_t float2_to_half2(float2 f) {
128
uint32_t u32;
129
uint16_t u16[2];
130
} tmp;
97
-
98
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
99
- asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
131
+#ifndef USE_ROCM
132
+ #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
133
+ asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
134
+ #else
135
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
136
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
137
+ #endif
138
#else
101
- asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
102
- asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
139
+ tmp.u16[0] = float_to_half(f.x);
140
+ tmp.u16[1] = float_to_half(f.y);
141
#endif
142
return tmp.u32;
143
}
@@ -107,13 +145,21 @@ inline __device__ uint32_t float2_to_half2(float2 f) {
145
// Vector addition.
146
inline __device__ uint16_t add(uint16_t a, uint16_t b) {
147
uint16_t c;
148
+#ifndef USE_ROCM
149
asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
150
+#else
151
+ asm volatile("v_add_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
152
+#endif
153
return c;
154
}
155
156
inline __device__ uint32_t add(uint32_t a, uint32_t b) {
157
uint32_t c;
158
+#ifndef USE_ROCM
159
asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
160
+#else
161
+ asm volatile("v_pk_add_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
162
+#endif
163
return c;
164
}
165
@@ -158,14 +204,22 @@ inline __device__ Float8_ add(uint4 a, Float8_ fb) {
204
template<>
205
inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
206
uint16_t c;
207
+#ifndef USE_ROCM
208
asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
209
+#else
210
+ asm volatile("v_mul_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
211
+#endif
212
return c;
213
}
214
215
template<>
216
inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
217
uint32_t c;
218
+#ifndef USE_ROCM
219
asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
220
+#else
221
+ asm volatile("v_pk_mul_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
222
+#endif
223
return c;
224
}
225
@@ -272,7 +326,11 @@ inline __device__ Float8_ mul(uint16_t a, uint4 b) {
326
// Vector fused multiply-add.
327
inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c) {
328
uint32_t d;
329
+#ifndef USE_ROCM
330
asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
331
+#else
332
+ asm volatile("v_pk_fma_f16 %0, %1, %2, %3;\n" : "=v"(d) : "v"(a), "v"(b), "v"(c));
333
+#endif
334
return d;
335
}
336
csrc/cache_kernels.cu
+7
-6
@@ -1,6 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
+#include "cuda_compat.h"
5
#include "dispatch_utils.h"
6
7
#include <algorithm>
@@ -28,8 +29,8 @@ void swap_blocks(
29
TORCH_CHECK(false, "Invalid device combination");
30
}
31
31
- void *src_ptr = src.data_ptr();
32
- void *dst_ptr = dst.data_ptr();
32
+ char *src_ptr = static_cast<char*>(src.data_ptr());
33
+ char *dst_ptr = static_cast<char*>(dst.data_ptr());
34
35
const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
36
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
@@ -267,8 +268,8 @@ __global__ void gather_cached_kv_kernel(
268
+ head_offset * block_size
269
+ block_offset;
270
270
- key[tgt_key_idx] = __ldg(&key_cache[src_key_idx]);
271
- value[tgt_value_idx] = __ldg(&value_cache[src_value_idx]);
271
+ key[tgt_key_idx] = VLLM_LDG(&key_cache[src_key_idx]);
272
+ value[tgt_value_idx] = VLLM_LDG(&value_cache[src_value_idx]);
273
}
274
}
275
@@ -333,8 +334,8 @@ __global__ void gather_cached_kv_kernel_optimized(
334
src_key_indices[j] = src_key_idx;
335
src_value_indices[j] = src_value_idx;
336
336
- keys_to_store[j] = __ldg(&key_cache[src_key_idx]);
337
- values_to_store[j] = __ldg(&value_cache[src_value_idx]);
337
+ keys_to_store[j] = VLLM_LDG(&key_cache[src_key_idx]);
338
+ values_to_store[j] = VLLM_LDG(&value_cache[src_value_idx]);
339
}
340
341
#pragma unroll
csrc/cuda_compat.h
new
+28
@@ -0,0 +1,28 @@
1
+#pragma once
2
+
3
+#ifndef USE_ROCM
4
+ #define VLLM_LDG(arg) __ldg(arg)
5
+#else
6
+ #define VLLM_LDG(arg) *(arg)
7
+#endif
8
+
9
+#ifndef USE_ROCM
10
+ #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor_sync(uint32_t(-1), var, lane_mask)
11
+#else
12
+ #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor(var, lane_mask)
13
+#endif
14
+
15
+#ifndef USE_ROCM
16
+ #define VLLM_SHFL_SYNC(var, src_lane) __shfl_sync(uint32_t(-1), var, src_lane)
17
+#else
18
+ #define VLLM_SHFL_SYNC(var, src_lane) __shfl(var, src_lane)
19
+#endif
20
+
21
+#ifndef USE_ROCM
22
+ #define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
23
+ cudaFuncSetAttribute(FUNC, cudaFuncAttributeMaxDynamicSharedMemorySize, VAL)
24
+#else
25
+ #define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
26
+ hipFuncSetAttribute(FUNC, hipFuncAttributeMaxDynamicSharedMemorySize, VAL)
27
+#endif
28
+
csrc/cuda_utils_kernels.cu
+3
@@ -1,3 +1,6 @@
1
+#ifdef USE_ROCM
2
+ #include <hip/hip_runtime.h>
3
+#endif
4
int get_device_attribute(
5
int attribute,
6
int device_id)
csrc/ops.h
+2
@@ -61,12 +61,14 @@ void gelu_fast(
61
torch::Tensor& out,
62
torch::Tensor& input);
63
64
+#ifndef USE_ROCM
65
torch::Tensor awq_gemm(
66
torch::Tensor _in_feats,
67
torch::Tensor _kernel,
68
torch::Tensor _scaling_factors,
69
torch::Tensor _zeros,
70
int split_k_iters);
71
+#endif
72
73
void squeezellm_gemm(
74
torch::Tensor vec,
csrc/pos_encoding_kernels.cu
+5
-4
@@ -1,6 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
+#include "cuda_compat.h"
5
#include "dispatch_utils.h"
6
7
namespace vllm {
@@ -19,14 +20,14 @@ inline __device__ void apply_rotary_embedding(
20
// GPT-NeoX style rotary embedding.
21
x_index = rot_offset;
22
y_index = embed_dim + rot_offset;
22
- cos = __ldg(cos_ptr + x_index);
23
- sin = __ldg(sin_ptr + x_index);
23
+ cos = VLLM_LDG(cos_ptr + x_index);
24
+ sin = VLLM_LDG(sin_ptr + x_index);
25
} else {
26
// GPT-J style rotary embedding.
27
x_index = 2 * rot_offset;
28
y_index = 2 * rot_offset + 1;
28
- cos = __ldg(cos_ptr + x_index / 2);
29
- sin = __ldg(sin_ptr + x_index / 2);
29
+ cos = VLLM_LDG(cos_ptr + x_index / 2);
30
+ sin = VLLM_LDG(sin_ptr + x_index / 2);
31
}
32
33
const scalar_t x = arr[x_index];
csrc/pybind.cpp
+4
@@ -48,8 +48,12 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
48
&rotary_embedding,
49
"Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
50
51
+#ifndef USE_ROCM
52
// Quantization ops
53
ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
54
+#endif
55
+
56
+
57
ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
58
59
// Cache ops
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+74
@@ -20,9 +20,17 @@ __device__ inline unsigned int as_unsigned(int i) {
20
21
// 4-bit matvec kernel (LUT-based)
22
__global__ void NUQ4MatMulKernel(
23
+#ifndef USE_ROCM
24
const half2* __restrict__ vec,
25
+#else
26
+ const __half2* __restrict__ vec,
27
+#endif
28
const int* __restrict__ mat,
29
+#ifndef USE_ROCM
30
half2* __restrict__ mul,
31
+#else
32
+ float2* __restrict__ mul,
33
+#endif
34
const __half* __restrict__ lookup_table,
35
int height,
36
int width,
@@ -35,7 +43,11 @@ __global__ void NUQ4MatMulKernel(
43
int row = BLOCKHEIGHT4 * blockIdx.x;
44
int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
45
46
+#ifndef USE_ROCM
47
__shared__ half2 blockvec[blockwidth2];
48
+#else
49
+ __shared__ __half2 blockvec[blockwidth2];
50
+#endif
51
52
__shared__ __half deq2[16][BLOCKWIDTH];
53
int off = threadIdx.x;
@@ -46,8 +58,13 @@ __global__ void NUQ4MatMulKernel(
58
}
59
60
__half res;
61
+#ifndef USE_ROCM
62
half2 res2;
63
half2 tmp2;
64
+#else
65
+ __half2 res2;
66
+ __half2 tmp2;
67
+#endif
68
69
int i;
70
int k;
@@ -68,48 +85,96 @@ __global__ void NUQ4MatMulKernel(
85
while (k < blockwidth2) {
86
tmp1 = as_unsigned(mat[i]);
87
88
+#ifndef USE_ROCM
89
res2 = {};
90
tmp2 = {};
91
+#else
92
+ res2.x = __half_as_ushort(__float2half(0));
93
+ res2.y = __half_as_ushort(__float2half(0));
94
+ tmp2.x = __half_as_ushort(__float2half(0));
95
+ tmp2.y = __half_as_ushort(__float2half(0));
96
+#endif
97
98
lut_index1 = tmp1 & 0xF;
99
lut_index2 = (tmp1 >> 4) & 0xF;
100
+#ifndef USE_ROCM
101
tmp2.x = deq2[lut_index1][off];
102
tmp2.y = deq2[lut_index2][off];
103
+#else
104
+ tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
105
+ tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
106
+#endif
107
res2 = __hfma2(tmp2, blockvec[k + 0], res2);
108
109
lut_index1 = (tmp1 >> 8) & 0xF;
110
lut_index2 = (tmp1 >> 12) & 0xF;
111
+#ifndef USE_ROCM
112
tmp2.x = deq2[lut_index1][off];
113
tmp2.y = deq2[lut_index2][off];
114
+#else
115
+ tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
116
+ tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
117
+#endif
118
res2 = __hfma2(tmp2, blockvec[k + 1], res2);
119
120
lut_index1 = (tmp1 >> 16) & 0xF;
121
lut_index2 = (tmp1 >> 20) & 0xF;
122
+#ifndef USE_ROCM
123
tmp2.x = deq2[lut_index1][off];
124
tmp2.y = deq2[lut_index2][off];
125
+#else
126
+ tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
127
+ tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
128
+#endif
129
res2 = __hfma2(tmp2, blockvec[k + 2], res2);
130
131
lut_index1 = (tmp1 >> 24) & 0xF;
132
lut_index2 = (tmp1 >> 28) & 0xF;
133
+#ifndef USE_ROCM
134
tmp2.x = deq2[lut_index1][off];
135
tmp2.y = deq2[lut_index2][off];
136
+#else
137
+ tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
138
+ tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
139
+#endif
140
res2 = __hfma2(tmp2, blockvec[k + 3], res2);
141
142
+#ifndef USE_ROCM
143
res = __hadd(__hadd(res2.x, res2.y), res);
144
+#else
145
+ res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)), res);
146
+#endif
147
148
i += width;
149
k += 4;
150
}
151
152
// col%2 -> only set one of the two values
153
+#ifndef USE_ROCM
154
half2 res3 = {};
155
if (col % 2 == 0) {
156
res3.x = res;
157
} else {
158
res3.y = res;
159
}
160
+#else
161
+ __half2 res3;
162
+ res3.x = __half_as_ushort(__float2half(0));
163
+ res3.y = __half_as_ushort(__float2half(0));
164
+ if (col % 2 == 0) {
165
+ res3.x = __half_as_ushort(res);
166
+ } else {
167
+ res3.y = __half_as_ushort(res);
168
+ }
169
+#endif
170
171
+#ifndef USE_ROCM
172
atomicAdd(&mul[b * width / 2 + col / 2], res3);
173
+#else
174
+ int tmp_addr = b * width / 2 + col / 2;
175
+ atomicAdd(&(mul[tmp_addr].x), __half2float(__ushort_as_half(res3.x)));
176
+ atomicAdd(&(mul[tmp_addr].y), __half2float(__ushort_as_half(res3.y)));
177
+#endif
178
}
179
}
180
@@ -136,10 +201,19 @@ void squeezellm_gemm(
201
dim3 threads(BLOCKWIDTH);
202
203
vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
204
+#ifndef USE_ROCM
205
(half2*) vec.data<at::Half>(),
206
+#else
207
+ (__half2*) vec.data_ptr<at::Half>(),
208
+#endif
209
mat.data_ptr<int>(),
210
+#ifndef USE_ROCM
211
(half2*) mul.data<at::Half>(),
212
(__half*) lookup_table.data<at::Half>(),
213
+#else
214
+ (float2*) mul.data_ptr<float>(),
215
+ (__half*) lookup_table.data_ptr<at::Half>(),
216
+#endif
217
height, width, batch, vec_height
218
);
219
}
csrc/reduction_utils.cuh
+3
-1
@@ -17,13 +17,15 @@
17
*/
18
#pragma once
19
20
+#include "cuda_compat.h"
21
+
22
namespace vllm {
23
24
template<typename T>
25
__inline__ __device__ T warpReduceSum(T val) {
26
#pragma unroll
27
for (int mask = 16; mask > 0; mask >>= 1)
26
- val += __shfl_xor_sync(0xffffffff, val, mask, 32);
28
+ val += VLLM_SHFL_XOR_SYNC(val, mask);
29
return val;
30
}
31
docs/source/getting_started/amd-installation.rst
new
+143
@@ -0,0 +1,143 @@
1
+.. _installation_rocm:
2
+
3
+Installation with ROCm
4
+======================
5
+
6
+vLLM 0.2.x onwards supports model inferencing and serving on AMD GPUs with ROCm.
7
+At the moment AWQ quantization is not supported in ROCm, but SqueezeLLM quantization has been ported.
8
+Data types currently supported in ROCm are FP16 and BF16.
9
+
10
+Requirements
11
+------------
12
+
13
+* OS: Linux
14
+* Python: 3.8 -- 3.11 (Verified on 3.10)
15
+* GPU: MI200s
16
+* Pytorch 2.0.1/2.1.1/2.2
17
+* ROCm 5.7
18
+
19
+Installation options:
20
+
21
+#. :ref:`(Recommended) Quick start with vLLM pre-installed in Docker Image <quick_start_docker_rocm>`
22
+#. :ref:`Build from source <build_from_source_rocm>`
23
+#. :ref:`Build from source with docker <build_from_source_docker_rocm>`
24
+
25
+.. _quick_start_docker_rocm:
26
+
27
+(Recommended) Option 1: Quick start with vLLM pre-installed in Docker Image
28
+---------------------------------------------------------------------------
29
+
30
+.. code-block:: console
31
+
32
+ $ docker pull embeddedllminfo/vllm-rocm:vllm-v0.2.3
33
+ $ docker run -it \
34
+ --network=host \
35
+ --group-add=video \
36
+ --ipc=host \
37
+ --cap-add=SYS_PTRACE \
38
+ --security-opt seccomp=unconfined \
39
+ --device /dev/kfd \
40
+ --device /dev/dri \
41
+ -v <path/to/model>:/app/model \
42
+ embeddedllminfo/vllm-rocm \
43
+ bash
44
+
45
+
46
+.. _build_from_source_rocm:
47
+
48
+Option 2: Build from source
49
+---------------------------
50
+
51
+You can build and install vLLM from source:
52
+
53
+0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
54
+
55
+- `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
56
+- `Pytorch <https://pytorch.org/>`_
57
+
58
+ .. code-block:: console
59
+
60
+ $ pip install torch==2.2.0.dev20231206+rocm5.7 --index-url https://download.pytorch.org/whl/nightly/rocm5.7 # tested version
61
+
62
+
63
+1. Install `flash attention for ROCm <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm>`_
64
+
65
+ Install ROCm's flash attention (v2.0.4) following the instructions from `ROCmSoftwarePlatform/flash-attention <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm#amd-gpurocm-support>`_
66
+
67
+.. note::
68
+ - If you are using rocm5.7 with pytorch 2.1.0 onwards, you don't need to apply the `hipify_python.patch`. You can build the ROCm flash attention directly.
69
+ - If you fail to install `ROCmSoftwarePlatform/flash-attention`, try cloning from the commit `6fd2f8e572805681cd67ef8596c7e2ce521ed3c6`.
70
+ - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
71
+ - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
72
+
73
+2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
74
+
75
+ .. code-block:: console
76
+
77
+ $ pip install xformers==0.0.22.post7 --no-deps
78
+ $ bash patch_xformers-0.0.22.post7.rocm.sh
79
+
80
+3. Build vLLM.
81
+
82
+ .. code-block:: console
83
+
84
+ $ cd vllm
85
+ $ pip install -U -r requirements-rocm.txt
86
+ $ python setup.py install # This may take 5-10 minutes. Currently, `pip install .`` does not work for ROCm installation
87
+
88
+
89
+.. _build_from_source_docker_rocm:
90
+
91
+Option 3: Build from source with docker
92
+-----------------------------------------------------
93
+
94
+You can build and install vLLM from source:
95
+
96
+Build a docker image from `Dockerfile.rocm`, and launch a docker container.
97
+
98
+.. code-block:: console
99
+
100
+ $ docker build -f Dockerfile.rocm -t vllm-rocm .
101
+ $ docker run -it \
102
+ --network=host \
103
+ --group-add=video \
104
+ --ipc=host \
105
+ --cap-add=SYS_PTRACE \
106
+ --security-opt seccomp=unconfined \
107
+ --device /dev/kfd \
108
+ --device /dev/dri \
109
+ -v <path/to/model>:/app/model \
110
+ vllm-rocm \
111
+ bash
112
+
113
+Alternatively, if you plan to install vLLM-ROCm on a local machine or start from a fresh docker image (e.g. rocm/pytorch), you can follow the steps below:
114
+
115
+0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
116
+
117
+- `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
118
+- `Pytorch <https://pytorch.org/>`_
119
+
120
+1. Install `flash attention for ROCm <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm>`_
121
+
122
+ Install ROCm's flash attention (v2.0.4) following the instructions from `ROCmSoftwarePlatform/flash-attention <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm#amd-gpurocm-support>`_
123
+
124
+.. note::
125
+ - If you are using rocm5.7 with pytorch 2.1.0 onwards, you don't need to apply the `hipify_python.patch`. You can build the ROCm flash attention directly.
126
+ - If you fail to install `ROCmSoftwarePlatform/flash-attention`, try cloning from the commit `6fd2f8e572805681cd67ef8596c7e2ce521ed3c6`.
127
+ - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
128
+ - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
129
+
130
+2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
131
+
132
+ .. code-block:: console
133
+
134
+ $ pip install xformers==0.0.22.post7 --no-deps
135
+ $ bash patch_xformers-0.0.22.post7.rocm.sh
136
+
137
+3. Build vLLM.
138
+
139
+ .. code-block:: console
140
+
141
+ $ cd vllm
142
+ $ pip install -U -r requirements-rocm.txt
143
+ $ python setup.py install # This may take 5-10 minutes.
docs/source/index.rst
+2
@@ -39,6 +39,7 @@ vLLM is flexible and easy to use with:
39
* Tensor parallelism support for distributed inference
40
* Streaming outputs
41
* OpenAI-compatible API server
42
+* Support NVIDIA CUDA and AMD ROCm.
43
44
For more information, check out the following:
45
@@ -56,6 +57,7 @@ Documentation
57
:caption: Getting Started
58
59
getting_started/installation
60
+ getting_started/amd-installation
61
getting_started/quickstart
62
63
.. toctree::
patch_xformers-0.0.22.post7.rocm.sh
new
+22
@@ -0,0 +1,22 @@
1
+#!/bin/bash
2
+export XFORMERS_FMHA_FLASH_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.flash.__file__)')
3
+export XFORMERS_FMHA_COMMON_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.common.__file__)')
4
+
5
+echo $XFORMERS_FMHA_FLASH_PATH
6
+echo $XFORMERS_FMHA_COMMON_PATH
7
+
8
+if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"; then
9
+ echo "Applying patch to ${XFORMERS_FMHA_FLASH_PATH}"
10
+ patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"
11
+ echo "Successfully patch ${XFORMERS_FMHA_FLASH_PATH}"
12
+else
13
+ echo "${XFORMERS_FMHA_FLASH_PATH} was patched before"
14
+fi
15
+
16
+if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"; then
17
+ echo "Applying patch to ${XFORMERS_FMHA_COMMON_PATH}"
18
+ patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"
19
+ echo "Successfully patch ${XFORMERS_FMHA_COMMON_PATH}"
20
+else
21
+ echo "${XFORMERS_FMHA_COMMON_PATH} was patched before"
22
+fi
requirements-rocm.txt
new
+17
@@ -0,0 +1,17 @@
1
+ninja # For faster builds.
2
+typing-extensions>=4.8.0
3
+starlette
4
+psutil
5
+ray >= 2.5.1
6
+pandas # Required for Ray data.
7
+pyarrow # Required for Ray data.
8
+sentencepiece # Required for LLaMA tokenizer.
9
+numpy
10
+tokenizers>=0.15.0
11
+huggingface_hub<0.18,>=0.16.4
12
+einops # Required for phi-1_5
13
+transformers >= 4.34.0 # Required for Mistral.
14
+fastapi
15
+uvicorn[standard]
16
+pydantic == 1.10.13 # Required for OpenAI server.
17
+aioprometheus[starlette]
rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch
new
+13
@@ -0,0 +1,13 @@
1
+--- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/common.py 2023-11-29 03:17:03.930103539 +0000
2
++++ common.py 2023-11-28 16:14:19.846233146 +0000
3
+@@ -298,8 +298,8 @@
4
+ dtype = d.query.dtype
5
+ if device_type not in cls.SUPPORTED_DEVICES:
6
+ reasons.append(f"device={device_type} (supported: {cls.SUPPORTED_DEVICES})")
7
+- if device_type == "cuda" and not _built_with_cuda:
8
+- reasons.append("xFormers wasn't build with CUDA support")
9
++ #if device_type == "cuda" and not _built_with_cuda:
10
++ # reasons.append("xFormers wasn't build with CUDA support")
11
+ if device_type == "cuda":
12
+ device_capability = torch.cuda.get_device_capability(d.device)
13
+ if device_capability < cls.CUDA_MINIMUM_COMPUTE_CAPABILITY:
rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch
new
+134
@@ -0,0 +1,134 @@
1
+--- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/flash.py 2023-11-29 03:17:03.930103539 +0000
2
++++ flash.py 2023-11-28 16:14:25.206128903 +0000
3
+@@ -31,39 +31,39 @@
4
+
5
+ FLASH_VERSION = "0.0.0"
6
+ try:
7
+- try:
8
+- from ... import _C_flashattention # type: ignore[attr-defined]
9
+- from ..._cpp_lib import _build_metadata
10
+-
11
+- if _build_metadata is not None:
12
+- FLASH_VERSION = _build_metadata.flash_version
13
+- except ImportError:
14
+- import flash_attn
15
+- from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
16
+-
17
+- FLASH_VERSION = flash_attn.__version__
18
+- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
19
+- if flash_ver_parsed < (2, 3):
20
+- raise ImportError("Requires 2.3 for sliding window support")
21
++ #try:
22
++ # from ... import _C_flashattention # type: ignore[attr-defined]
23
++ # from ..._cpp_lib import _build_metadata
24
++
25
++ # if _build_metadata is not None:
26
++ # FLASH_VERSION = _build_metadata.flash_version
27
++ #except ImportError:
28
++ import flash_attn
29
++ from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
30
++
31
++ FLASH_VERSION = flash_attn.__version__
32
++ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
33
++ # if flash_ver_parsed < (2, 3):
34
++ # raise ImportError("Requires 2.3 for sliding window support")
35
+
36
+ # create library so that flash-attn goes through the PyTorch Dispatcher
37
+- _flash_lib = torch.library.Library("xformers_flash", "DEF")
38
++ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
39
+
40
+- _flash_lib.define(
41
+- "flash_fwd(Tensor query, Tensor key, Tensor value, "
42
+- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
43
+- "int max_seqlen_q, int max_seqlen_k, "
44
+- "float p, float softmax_scale, "
45
+- "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
46
+- )
47
+-
48
+- _flash_lib.define(
49
+- "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
50
+- "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
51
+- "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
52
+- "int max_seqlen_q, int max_seqlen_k, "
53
+- "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
54
+- )
55
++ #_flash_lib.define(
56
++ # "flash_fwd(Tensor query, Tensor key, Tensor value, "
57
++ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
58
++ # "int max_seqlen_q, int max_seqlen_k, "
59
++ # "float p, float softmax_scale, "
60
++ # "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
61
++ #)
62
++
63
++ #_flash_lib.define(
64
++ # "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
65
++ # "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
66
++ # "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
67
++ # "int max_seqlen_q, int max_seqlen_k, "
68
++ # "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
69
++ #)
70
+
71
+ def _flash_fwd(
72
+ query,
73
+@@ -98,8 +98,8 @@
74
+ p,
75
+ softmax_scale,
76
+ is_causal,
77
+- window_size - 1, # window_size_left
78
+- -1, # window_size_right
79
++ # window_size - 1, # window_size_left
80
++ # -1, # window_size_right
81
+ return_softmax,
82
+ None, # rng
83
+ )
84
+@@ -127,8 +127,8 @@
85
+ softmax_scale,
86
+ False,
87
+ is_causal,
88
+- window_size - 1, # window_size_left
89
+- -1, # window_size_right
90
++ # window_size - 1, # window_size_left
91
++ # -1, # window_size_right
92
+ return_softmax,
93
+ None,
94
+ )
95
+@@ -169,8 +169,8 @@
96
+ p,
97
+ softmax_scale,
98
+ is_causal,
99
+- window_size - 1, # window_size_left
100
+- -1, # window_size_right
101
++ # window_size - 1, # window_size_left
102
++ # -1, # window_size_right
103
+ None,
104
+ rng_state,
105
+ )
106
+@@ -193,15 +193,15 @@
107
+ softmax_scale,
108
+ False, # zero_tensors
109
+ is_causal,
110
+- window_size - 1, # window_size_left
111
+- -1, # window_size_right
112
++ # window_size - 1, # window_size_left
113
++ # -1, # window_size_right
114
+ None,
115
+ rng_state,
116
+ )
117
+ return dq, dk, dv
118
+
119
+- _flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
120
+- _flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
121
++ #_flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
122
++ #_flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
123
+ except ImportError:
124
+ pass
125
+
126
+@@ -348,7 +348,7 @@
127
+ implementation.
128
+ """
129
+
130
+- OPERATOR = get_operator("xformers_flash", "flash_fwd")
131
++ OPERATOR = _flash_fwd # get_operator("xformers_flash", "flash_fwd")
132
+ SUPPORTED_DEVICES: Set[str] = {"cuda"}
133
+ CUDA_MINIMUM_COMPUTE_CAPABILITY = (8, 0)
134
+ SUPPORTED_DTYPES: Set[torch.dtype] = {torch.half, torch.bfloat16}
setup.py
+159
-73
@@ -8,27 +8,83 @@ import warnings
8
from packaging.version import parse, Version
9
import setuptools
10
import torch
11
-from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
11
+from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
12
13
ROOT_DIR = os.path.dirname(__file__)
14
15
MAIN_CUDA_VERSION = "12.1"
16
17
# Supported NVIDIA GPU architectures.
18
-SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
18
+NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
19
+ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
20
+# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
21
+
22
+
23
+def _is_hip() -> bool:
24
+ return torch.version.hip is not None
25
+
26
+
27
+def _is_cuda() -> bool:
28
+ return torch.version.cuda is not None
29
+
30
31
# Compiler flags.
32
CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
33
# TODO(woosuk): Should we use -O3?
34
NVCC_FLAGS = ["-O2", "-std=c++17"]
35
36
+if _is_hip():
37
+ if ROCM_HOME is None:
38
+ raise RuntimeError(
39
+ "Cannot find ROCM_HOME. ROCm must be available to build the package."
40
+ )
41
+ NVCC_FLAGS += ["-DUSE_ROCM"]
42
+
43
+if _is_cuda() and CUDA_HOME is None:
44
+ raise RuntimeError(
45
+ "Cannot find CUDA_HOME. CUDA must be available to build the package.")
46
+
47
ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
48
CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
49
NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
50
29
-if CUDA_HOME is None:
30
- raise RuntimeError(
31
- "Cannot find CUDA_HOME. CUDA must be available to build the package.")
51
+
52
+def get_amdgpu_offload_arch():
53
+ command = "/opt/rocm/llvm/bin/amdgpu-offload-arch"
54
+ try:
55
+ output = subprocess.check_output([command])
56
+ return output.decode('utf-8').strip()
57
+ except subprocess.CalledProcessError as e:
58
+ error_message = f"Error: {e}"
59
+ raise RuntimeError(error_message) from e
60
+ except FileNotFoundError as e:
61
+ # If the command is not found, print an error message
62
+ error_message = f"The command {command} was not found."
63
+ raise RuntimeError(error_message) from e
64
+
65
+ return None
66
+
67
+
68
+def get_hipcc_rocm_version():
69
+ # Run the hipcc --version command
70
+ result = subprocess.run(['hipcc', '--version'],
71
+ stdout=subprocess.PIPE,
72
+ stderr=subprocess.STDOUT,
73
+ text=True)
74
+
75
+ # Check if the command was executed successfully
76
+ if result.returncode != 0:
77
+ print("Error running 'hipcc --version'")
78
+ return None
79
+
80
+ # Extract the version using a regular expression
81
+ match = re.search(r'HIP version: (\S+)', result.stdout)
82
+ if match:
83
+ # Return the version string
84
+ return match.group(1)
85
+ else:
86
+ print("Could not find HIP version in the output")
87
+ return None
88
89
90
def get_nvcc_cuda_version(cuda_dir: str) -> Version:
@@ -61,20 +117,22 @@ def get_torch_arch_list() -> Set[str]:
117
return set()
118
119
# Filter out the invalid architectures and print a warning.
64
- valid_archs = SUPPORTED_ARCHS.union({s + "+PTX" for s in SUPPORTED_ARCHS})
120
+ valid_archs = NVIDIA_SUPPORTED_ARCHS.union(
121
+ {s + "+PTX"
122
+ for s in NVIDIA_SUPPORTED_ARCHS})
123
arch_list = torch_arch_list.intersection(valid_archs)
124
# If none of the specified architectures are valid, raise an error.
125
if not arch_list:
126
raise RuntimeError(
69
- "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
127
+ "None of the CUDA/ROCM architectures in `TORCH_CUDA_ARCH_LIST` env "
128
f"variable ({env_arch_list}) is supported. "
71
- f"Supported CUDA architectures are: {valid_archs}.")
129
+ f"Supported CUDA/ROCM architectures are: {valid_archs}.")
130
invalid_arch_list = torch_arch_list - valid_archs
131
if invalid_arch_list:
132
warnings.warn(
75
- f"Unsupported CUDA architectures ({invalid_arch_list}) are "
133
+ f"Unsupported CUDA/ROCM architectures ({invalid_arch_list}) are "
134
"excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
77
- f"({env_arch_list}). Supported CUDA architectures are: "
135
+ f"({env_arch_list}). Supported CUDA/ROCM architectures are: "
136
f"{valid_archs}.",
137
stacklevel=2)
138
return arch_list
@@ -82,7 +140,7 @@ def get_torch_arch_list() -> Set[str]:
140
141
# First, check the TORCH_CUDA_ARCH_LIST environment variable.
142
compute_capabilities = get_torch_arch_list()
85
-if not compute_capabilities:
143
+if _is_cuda() and not compute_capabilities:
144
# If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
145
# GPUs on the current machine.
146
device_count = torch.cuda.device_count()
@@ -93,69 +151,84 @@ if not compute_capabilities:
151
"GPUs with compute capability below 7.0 are not supported.")
152
compute_capabilities.add(f"{major}.{minor}")
153
96
-nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
97
-if not compute_capabilities:
98
- # If no GPU is specified nor available, add all supported architectures
99
- # based on the NVCC CUDA version.
100
- compute_capabilities = SUPPORTED_ARCHS.copy()
101
- if nvcc_cuda_version < Version("11.1"):
102
- compute_capabilities.remove("8.6")
154
+if _is_cuda():
155
+ nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
156
+ if not compute_capabilities:
157
+ # If no GPU is specified nor available, add all supported architectures
158
+ # based on the NVCC CUDA version.
159
+ compute_capabilities = NVIDIA_SUPPORTED_ARCHS.copy()
160
+ if nvcc_cuda_version < Version("11.1"):
161
+ compute_capabilities.remove("8.6")
162
+ if nvcc_cuda_version < Version("11.8"):
163
+ compute_capabilities.remove("8.9")
164
+ compute_capabilities.remove("9.0")
165
+ # Validate the NVCC CUDA version.
166
+ if nvcc_cuda_version < Version("11.0"):
167
+ raise RuntimeError(
168
+ "CUDA 11.0 or higher is required to build the package.")
169
+ if (nvcc_cuda_version < Version("11.1")
170
+ and any(cc.startswith("8.6") for cc in compute_capabilities)):
171
+ raise RuntimeError(
172
+ "CUDA 11.1 or higher is required for compute capability 8.6.")
173
if nvcc_cuda_version < Version("11.8"):
104
- compute_capabilities.remove("8.9")
105
- compute_capabilities.remove("9.0")
106
-
107
-# Validate the NVCC CUDA version.
108
-if nvcc_cuda_version < Version("11.0"):
109
- raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
110
-if (nvcc_cuda_version < Version("11.1")
111
- and any(cc.startswith("8.6") for cc in compute_capabilities)):
112
- raise RuntimeError(
113
- "CUDA 11.1 or higher is required for compute capability 8.6.")
114
-if nvcc_cuda_version < Version("11.8"):
115
- if any(cc.startswith("8.9") for cc in compute_capabilities):
116
- # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
117
- # However, GPUs with compute capability 8.9 can also run the code generated by
118
- # the previous versions of CUDA 11 and targeting compute capability 8.0.
119
- # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
120
- # instead of 8.9.
121
- warnings.warn(
122
- "CUDA 11.8 or higher is required for compute capability 8.9. "
123
- "Targeting compute capability 8.0 instead.",
124
- stacklevel=2)
125
- compute_capabilities = set(cc for cc in compute_capabilities
126
- if not cc.startswith("8.9"))
127
- compute_capabilities.add("8.0+PTX")
128
- if any(cc.startswith("9.0") for cc in compute_capabilities):
174
+ if any(cc.startswith("8.9") for cc in compute_capabilities):
175
+ # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
176
+ # However, GPUs with compute capability 8.9 can also run the code generated by
177
+ # the previous versions of CUDA 11 and targeting compute capability 8.0.
178
+ # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
179
+ # instead of 8.9.
180
+ warnings.warn(
181
+ "CUDA 11.8 or higher is required for compute capability 8.9. "
182
+ "Targeting compute capability 8.0 instead.",
183
+ stacklevel=2)
184
+ compute_capabilities = set(cc for cc in compute_capabilities
185
+ if not cc.startswith("8.9"))
186
+ compute_capabilities.add("8.0+PTX")
187
+ if any(cc.startswith("9.0") for cc in compute_capabilities):
188
+ raise RuntimeError(
189
+ "CUDA 11.8 or higher is required for compute capability 9.0.")
190
+
191
+ # Add target compute capabilities to NVCC flags.
192
+ for capability in compute_capabilities:
193
+ num = capability[0] + capability[2]
194
+ NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
195
+ if capability.endswith("+PTX"):
196
+ NVCC_FLAGS += [
197
+ "-gencode", f"arch=compute_{num},code=compute_{num}"
198
+ ]
199
+
200
+ # Use NVCC threads to parallelize the build.
201
+ if nvcc_cuda_version >= Version("11.2"):
202
+ nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
203
+ num_threads = min(os.cpu_count(), nvcc_threads)
204
+ NVCC_FLAGS += ["--threads", str(num_threads)]
205
+
206
+elif _is_hip():
207
+ amd_arch = get_amdgpu_offload_arch()
208
+ if amd_arch not in ROCM_SUPPORTED_ARCHS:
209
raise RuntimeError(
130
- "CUDA 11.8 or higher is required for compute capability 9.0.")
210
+ f"Only the following arch is supported: {ROCM_SUPPORTED_ARCHS}"
211
+ f"amdgpu_arch_found: {amd_arch}")
212
+
213
+ext_modules = []
214
132
-# Add target compute capabilities to NVCC flags.
133
-for capability in compute_capabilities:
134
- num = capability[0] + capability[2]
135
- NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
136
- if capability.endswith("+PTX"):
137
- NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=compute_{num}"]
215
+vllm_extension_sources = [
216
+ "csrc/cache_kernels.cu",
217
+ "csrc/attention/attention_kernels.cu",
218
+ "csrc/pos_encoding_kernels.cu",
219
+ "csrc/activation_kernels.cu",
220
+ "csrc/layernorm_kernels.cu",
221
+ "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
222
+ "csrc/cuda_utils_kernels.cu",
223
+ "csrc/pybind.cpp",
224
+]
225
139
-# Use NVCC threads to parallelize the build.
140
-if nvcc_cuda_version >= Version("11.2"):
141
- nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
142
- num_threads = min(os.cpu_count(), nvcc_threads)
143
- NVCC_FLAGS += ["--threads", str(num_threads)]
226
+if _is_cuda():
227
+ vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
228
145
-ext_modules = []
229
vllm_extension = CUDAExtension(
230
name="vllm._C",
148
- sources=[
149
- "csrc/cache_kernels.cu",
150
- "csrc/attention/attention_kernels.cu",
151
- "csrc/pos_encoding_kernels.cu",
152
- "csrc/activation_kernels.cu",
153
- "csrc/layernorm_kernels.cu",
154
- "csrc/quantization/awq/gemm_kernels.cu",
155
- "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
156
- "csrc/cuda_utils_kernels.cu",
157
- "csrc/pybind.cpp",
158
- ],
231
+ sources=vllm_extension_sources,
232
extra_compile_args={
233
"cxx": CXX_FLAGS,
234
"nvcc": NVCC_FLAGS,
@@ -183,10 +256,19 @@ def find_version(filepath: str) -> str:
256
257
def get_vllm_version() -> str:
258
version = find_version(get_path("vllm", "__init__.py"))
186
- cuda_version = str(nvcc_cuda_version)
187
- if cuda_version != MAIN_CUDA_VERSION:
188
- cuda_version_str = cuda_version.replace(".", "")[:3]
189
- version += f"+cu{cuda_version_str}"
259
+
260
+ if _is_hip():
261
+ # Get the HIP version
262
+ hipcc_version = get_hipcc_rocm_version()
263
+ if hipcc_version != MAIN_CUDA_VERSION:
264
+ rocm_version_str = hipcc_version.replace(".", "")[:3]
265
+ version += f"+rocm{rocm_version_str}"
266
+ else:
267
+ cuda_version = str(nvcc_cuda_version)
268
+ if cuda_version != MAIN_CUDA_VERSION:
269
+ cuda_version_str = cuda_version.replace(".", "")[:3]
270
+ version += f"+cu{cuda_version_str}"
271
+
272
return version
273
274
@@ -201,8 +283,12 @@ def read_readme() -> str:
283
284
def get_requirements() -> List[str]:
285
"""Get Python package dependencies from requirements.txt."""
204
- with open(get_path("requirements.txt")) as f:
205
- requirements = f.read().strip().split("\n")
286
+ if _is_hip():
287
+ with open(get_path("requirements-rocm.txt")) as f:
288
+ requirements = f.read().strip().split("\n")
289
+ else:
290
+ with open(get_path("requirements.txt")) as f:
291
+ requirements = f.read().strip().split("\n")
292
return requirements
293
294
vllm/config.py
+35
-4
@@ -6,7 +6,7 @@ from transformers import PretrainedConfig
6
7
from vllm.logger import init_logger
8
from vllm.transformers_utils.config import get_config
9
-from vllm.utils import get_cpu_memory
9
+from vllm.utils import get_cpu_memory, is_hip
10
11
logger = init_logger(__name__)
12
@@ -98,12 +98,27 @@ class ModelConfig:
98
99
def _verify_load_format(self) -> None:
100
load_format = self.load_format.lower()
101
- if load_format not in [
102
- "auto", "pt", "safetensors", "npcache", "dummy"
103
- ]:
101
+ supported_load_format = [
102
+ "auto", "pt", "safetensors", "npcache", "dummy"
103
+ ]
104
+ rocm_not_supported_load_format = ["safetensors"]
105
+ if load_format not in supported_load_format:
106
raise ValueError(
107
f"Unknown load format: {self.load_format}. Must be one of "
108
"'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
109
+ if is_hip():
110
+ if load_format in ["safetensors"]:
111
+ rocm_supported_load_format = [
112
+ f for f in supported_load_format
113
+ if (f not in rocm_not_supported_load_format)
114
+ ]
115
+ raise ValueError(
116
+ f"load format \'{load_format}\' is not supported in ROCm. "
117
+ f"Supported load format are "
118
+ f"{rocm_supported_load_format}")
119
+ # Force ROCm to load from pt weights if nothing specific is set
120
+ if load_format == "auto":
121
+ load_format = "pt"
122
self.load_format = load_format
123
124
def _verify_tokenizer_mode(self) -> None:
@@ -116,6 +131,7 @@ class ModelConfig:
131
132
def _verify_quantization(self) -> None:
133
supported_quantization = ["awq", "squeezellm"]
134
+ rocm_not_supported_quantization = ["awq"]
135
if self.quantization is not None:
136
self.quantization = self.quantization.lower()
137
@@ -137,6 +153,11 @@ class ModelConfig:
153
raise ValueError(
154
f"Unknown quantization method: {self.quantization}. Must "
155
f"be one of {supported_quantization}.")
156
+ if is_hip(
157
+ ) and self.quantization in rocm_not_supported_quantization:
158
+ raise ValueError(
159
+ f"{self.quantization} quantization is currently not supported "
160
+ f"in ROCm.")
161
logger.warning(f"{self.quantization} quantization is not fully "
162
"optimized yet. The speed can be slower than "
163
"non-quantized models.")
@@ -364,6 +385,8 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
385
"bfloat16": torch.bfloat16,
386
}
387
388
+_ROCM_NOT_SUPPORTED_DTYPE = ["float", "float32"]
389
+
390
391
def _get_and_verify_dtype(
392
config: PretrainedConfig,
@@ -393,6 +416,14 @@ def _get_and_verify_dtype(
416
else:
417
raise ValueError(f"Unknown dtype: {dtype}")
418
419
+ if is_hip() and torch_dtype == torch.float32:
420
+ rocm_supported_dtypes = [
421
+ k for k, v in _STR_DTYPE_TO_TORCH_DTYPE.items()
422
+ if (k not in _ROCM_NOT_SUPPORTED_DTYPE)
423
+ ]
424
+ raise ValueError(f"dtype \'{dtype}\' is not supported in ROCm. "
425
+ f"Supported dtypes are {rocm_supported_dtypes}")
426
+
427
# Verify the dtype.
428
if torch_dtype != config_dtype:
429
if torch_dtype == torch.float32:
vllm/engine/ray_utils.py
+7
-1
@@ -3,6 +3,7 @@ from typing import Optional, Tuple, TYPE_CHECKING
3
4
from vllm.config import ParallelConfig
5
from vllm.logger import init_logger
6
+from vllm.utils import is_hip
7
8
logger = init_logger(__name__)
9
@@ -73,7 +74,12 @@ def initialize_cluster(
74
"Ray is not installed. Please install Ray to use distributed "
75
"serving.")
76
# Connect to a ray cluster.
76
- ray.init(address=ray_address, ignore_reinit_error=True)
77
+ if is_hip():
78
+ ray.init(address=ray_address,
79
+ ignore_reinit_error=True,
80
+ num_gpus=parallel_config.world_size)
81
+ else:
82
+ ray.init(address=ray_address, ignore_reinit_error=True)
83
84
if not parallel_config.worker_use_ray:
85
# Initialize cluster locally.
vllm/model_executor/layers/attention.py
+3
@@ -10,6 +10,7 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
10
from vllm._C import ops
11
from vllm._C import cache_ops
12
from vllm.model_executor.input_metadata import InputMetadata
13
+from vllm.utils import is_hip
14
15
_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
16
# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -160,6 +161,8 @@ class PagedAttention(nn.Module):
161
attn_bias=input_metadata.attn_bias,
162
p=0.0,
163
scale=self.scale,
164
+ op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
165
+ (is_hip()) else None,
166
)
167
output = out.view_as(query)
168
else:
vllm/model_executor/layers/quantization/squeezellm.py
+9
-3
@@ -7,6 +7,7 @@ from vllm._C import ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
10
+from vllm.utils import is_hip
11
12
13
class SqueezeLLMConfig(QuantizationConfig):
@@ -114,9 +115,14 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
115
lookup_table = weights["lookup_table"]
116
out_shape = x.shape[:-1] + (qweight.shape[-1], )
117
reshaped_x = x.reshape(-1, x.shape[-1])
117
- # NOTE: The output tensor should be zero-initialized.
118
- out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
119
- ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
118
+ if is_hip():
119
+ out_f = torch.zeros(out_shape, device="cuda", dtype=torch.float)
120
+ ops.squeezellm_gemm(reshaped_x, qweight, out_f, lookup_table)
121
+ out = out_f.to(dtype=torch.float16)
122
+ else:
123
+ # NOTE: The output tensor should be zero-initialized.
124
+ out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
125
+ ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
126
127
if bias is not None:
128
out = out + bias
vllm/model_executor/model_loader.py
+24
@@ -10,6 +10,10 @@ from vllm.config import ModelConfig
10
from vllm.model_executor.models import *
11
from vllm.model_executor.weight_utils import (get_quant_config,
12
initialize_dummy_weights)
13
+from vllm.utils import is_hip
14
+from vllm.logger import init_logger
15
+
16
+logger = init_logger(__name__)
17
18
# TODO(woosuk): Lazy-load the model classes.
19
_MODEL_REGISTRY = {
@@ -39,6 +43,18 @@ _MODEL_REGISTRY = {
43
"YiForCausalLM": YiForCausalLM,
44
}
45
46
+# Models to be disabled in ROCm
47
+_ROCM_UNSUPPORTED_MODELS = []
48
+if is_hip():
49
+ for rocm_model in _ROCM_UNSUPPORTED_MODELS:
50
+ del _MODEL_REGISTRY[rocm_model]
51
+
52
+# Models partially supported in ROCm
53
+_ROCM_PARTIALLY_SUPPORTED_MODELS = {
54
+ "MistralForCausalLM":
55
+ "Sliding window attention is not supported in ROCm's flash attention",
56
+}
57
+
58
59
@contextlib.contextmanager
60
def _set_default_torch_dtype(dtype: torch.dtype):
@@ -53,7 +69,15 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
69
architectures = getattr(config, "architectures", [])
70
for arch in architectures:
71
if arch in _MODEL_REGISTRY:
72
+ if is_hip() and arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
73
+ logger.warning(
74
+ f"{arch} is not fully supported in ROCm. Reason: "
75
+ f"{_ROCM_PARTIALLY_SUPPORTED_MODELS[arch]}")
76
return _MODEL_REGISTRY[arch]
77
+ elif arch in _ROCM_UNSUPPORTED_MODELS:
78
+ raise ValueError(
79
+ f"Model architecture {arch} is not supported by ROCm for now. \n"
80
+ f"Supported architectures {list(_MODEL_REGISTRY.keys())}")
81
raise ValueError(
82
f"Model architectures {architectures} are not supported for now. "
83
f"Supported architectures: {list(_MODEL_REGISTRY.keys())}")
vllm/utils.py
+5
-1
@@ -27,10 +27,14 @@ class Counter:
27
self.counter = 0
28
29
30
+def is_hip() -> bool:
31
+ return torch.version.hip is not None
32
+
33
+
34
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
35
"""Returns the maximum shared memory per thread block in bytes."""
36
# https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
33
- cudaDevAttrMaxSharedMemoryPerBlockOptin = 97
37
+ cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
38
max_shared_mem = cuda_utils.get_device_attribute(
39
cudaDevAttrMaxSharedMemoryPerBlockOptin, gpu)
40
return int(max_shared_mem)