@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ccc0bfffb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Merge EmbeddedLLM/vllm-rocm into vLLM main (#1836)

Co-authored-by: Philipp Moritz <pcmoritz@gmail.com> Co-authored-by: Amir Balwel <amoooori04@gmail.com> Co-authored-by: root <kuanfu.liu@akirakan.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: kuanfu <kuanfu.liu@embeddedllm.com> Co-authored-by: miloice <17350011+kliuae@users.noreply.github.com>

TJian committed Dec 8, 2023 at 15:16 UTC 6ccc0bfffbcf1b7e927cc3dcf4159fc74ff94d40
29 files changed +873 -118
.gitignore
+4
@@ -177,3 +177,7 @@ _build/
177 # vim swap files
178 *.swo
179 *.swp
180 +
181 +# hip files generated by PyTorch
182 +*.hip
183 +*_hip*
Dockerfile.rocm new
+62
@@ -0,0 +1,62 @@
1 +FROM rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1
2 +
3 +# Install some basic utilities
4 +RUN apt-get update && apt-get install python3 python3-pip -y
5 +
6 +# Install some basic utilities
7 +RUN apt-get update && apt-get install -y \
8 + curl \
9 + ca-certificates \
10 + sudo \
11 + git \
12 + bzip2 \
13 + libx11-6 \
14 + build-essential \
15 + wget \
16 + unzip \
17 + nvidia-cuda-toolkit \
18 + tmux \
19 + && rm -rf /var/lib/apt/lists/*
20 +
21 +### Mount Point ###
22 +# When launching the container, mount the code directory to /app
23 +ARG APP_MOUNT=/app
24 +VOLUME [ ${APP_MOUNT} ]
25 +WORKDIR ${APP_MOUNT}
26 +
27 +RUN python3 -m pip install --upgrade pip
28 +RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
29 +
30 +ENV LLVM_SYMBOLIZER_PATH=/opt/rocm/llvm/bin/llvm-symbolizer
31 +ENV PATH=$PATH:/opt/rocm/bin:/libtorch/bin:
32 +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
33 +ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
34 +
35 +# Install ROCm flash-attention
36 +RUN mkdir libs \
37 + && cd libs \
38 + && git clone https://github.com/ROCmSoftwarePlatform/flash-attention.git \
39 + && cd flash-attention \
40 + && git checkout 3d2b6f5 \
41 + && git submodule update --init \
42 + && export GPU_ARCHS=$(/opt/rocm/llvm/bin/amdgpu-offload-arch) \
43 + && patch /opt/conda/envs/py_3.10/lib/python3.10/site-packages/torch/utils/hipify/hipify_python.py hipify_patch.patch \
44 + && python3 setup.py install \
45 + && cd ..
46 +
47 +COPY ./ /app/vllm
48 +
49 +RUN python3 -m pip install --upgrade pip
50 +RUN pip install xformers==0.0.22.post7 --no-deps
51 +
52 +RUN cd /app \
53 + && cd vllm \
54 + && pip install -U -r requirements-rocm.txt \
55 + && bash patch_xformers-0.0.22.post7.rocm.sh \
56 + && python3 setup.py install \
57 + && cd ..
58 +
59 +RUN python3 -m pip install --upgrade pip
60 +RUN python3 -m pip install --no-cache-dir ray[all]
61 +
62 +CMD ["/bin/bash"]
README.md
+2
@@ -17,6 +17,7 @@ Easy, fast, and cheap LLM serving for everyone
17 ---
18
19 *Latest News* 🔥
20 +- [2023/12] Added ROCm support to vLLM.
21 - [2023/10] We hosted [the first vLLM meetup](https://lu.ma/first-vllm-meetup) in SF! Please find the meetup slides [here](https://docs.google.com/presentation/d/1QL-XPFXiFpDBh86DbEegFXBXFXjix4v032GhShbKf3s/edit?usp=sharing).
22 - [2023/09] We created our [Discord server](https://discord.gg/jz7wjKhh6g)! Join us to discuss vLLM and LLM serving! We will also post the latest announcements and updates there.
23 - [2023/09] We released our [PagedAttention paper](https://arxiv.org/abs/2309.06180) on arXiv!
@@ -43,6 +44,7 @@ vLLM is flexible and easy to use with:
44 - Tensor parallelism support for distributed inference
45 - Streaming outputs
46 - OpenAI-compatible API server
47 +- Support NVIDIA CUDA and AMD ROCm.
48
49 vLLM seamlessly supports many Hugging Face models, including the following architectures:
50
csrc/activation_kernels.cu
+4 -3
@@ -1,6 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 +#include "cuda_compat.h"
5 #include "dispatch_utils.h"
6
7 namespace vllm {
@@ -18,8 +19,8 @@ __global__ void silu_and_mul_kernel(
19 const int d) {
20 const int64_t token_idx = blockIdx.x;
21 for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
21 - const scalar_t x = __ldg(&input[token_idx * 2 * d + idx]);
22 - const scalar_t y = __ldg(&input[token_idx * 2 * d + d + idx]);
22 + const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
23 + const scalar_t y = VLLM_LDG(&input[token_idx * 2 * d + d + idx]);
24 out[token_idx * d + idx] = silu(x) * y;
25 }
26 }
@@ -57,7 +58,7 @@ __global__ void activation_kernel(
58 const int d) {
59 const int64_t token_idx = blockIdx.x;
60 for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
60 - const scalar_t x = __ldg(&input[token_idx * d + idx]);
61 + const scalar_t x = VLLM_LDG(&input[token_idx * d + idx]);
62 out[token_idx * d + idx] = ACT_FN(x);
63 }
64 }
csrc/attention/attention_kernels.cu
+21 -13
@@ -15,6 +15,10 @@
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 +#ifdef USE_ROCM
19 +#include <hip/hip_runtime.h>
20 +#endif
21 +
22 #include <torch/extension.h>
23 #include <ATen/cuda/CUDAContext.h>
24
@@ -23,7 +27,11 @@
27
28 #include <algorithm>
29
30 +#ifndef USE_ROCM
31 #define WARP_SIZE 32
32 +#else
33 +#define WARP_SIZE warpSize
34 +#endif
35 #define MAX(a, b) ((a) > (b) ? (a) : (b))
36 #define MIN(a, b) ((a) < (b) ? (a) : (b))
37 #define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
@@ -40,7 +48,7 @@ inline __device__ float block_sum(float* red_smem, float sum) {
48 // Compute the sum per warp.
49 #pragma unroll
50 for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
43 - sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
51 + sum += VLLM_SHFL_XOR_SYNC(sum, mask);
52 }
53
54 // Warp leaders store the data to shared memory.
@@ -59,11 +67,11 @@ inline __device__ float block_sum(float* red_smem, float sum) {
67 // Parallel reduction inside the warp.
68 #pragma unroll
69 for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
62 - sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
70 + sum += VLLM_SHFL_XOR_SYNC(sum, mask);
71 }
72
73 // Broadcast to other threads.
66 - return __shfl_sync(uint32_t(-1), sum, 0);
74 + return VLLM_SHFL_SYNC(sum, 0);
75 }
76
77 // TODO(woosuk): Merge the last two dimensions of the grid.
@@ -223,7 +231,7 @@ __device__ void paged_attention_kernel(
231 // The 0-th thread of each thread group already has its max qk value.
232 #pragma unroll
233 for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
226 - qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
234 + qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
235 }
236 if (lane == 0) {
237 red_smem[warp_idx] = qk_max;
@@ -235,10 +243,10 @@ __device__ void paged_attention_kernel(
243 qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
244 #pragma unroll
245 for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
238 - qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
246 + qk_max = fmaxf(qk_max, VLLM_SHFL_XOR_SYNC(qk_max, mask));
247 }
248 // Broadcast the max qk value to all threads.
241 - qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
249 + qk_max = VLLM_SHFL_SYNC(qk_max, 0);
250
251 // Get the sum of the exp values.
252 float exp_sum = 0.f;
@@ -326,7 +334,7 @@ __device__ void paged_attention_kernel(
334 float acc = accs[i];
335 #pragma unroll
336 for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
329 - acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
337 + acc += VLLM_SHFL_XOR_SYNC(acc, mask);
338 }
339 accs[i] = acc;
340 }
@@ -492,7 +500,7 @@ __global__ void paged_attention_v2_reduce_kernel(
500 // Reduce within the warp.
501 #pragma unroll
502 for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
495 - max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
503 + max_logit = fmaxf(max_logit, VLLM_SHFL_XOR_SYNC(max_logit, mask));
504 }
505 if (lane == 0) {
506 red_smem[warp_idx] = max_logit;
@@ -502,10 +510,10 @@ __global__ void paged_attention_v2_reduce_kernel(
510 max_logit = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
511 #pragma unroll
512 for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
505 - max_logit = fmaxf(max_logit, __shfl_xor_sync(uint32_t(-1), max_logit, mask));
513 + max_logit = fmaxf(max_logit, VLLM_SHFL_XOR_SYNC(max_logit, mask));
514 }
515 // Broadcast the max value to all threads.
508 - max_logit = __shfl_sync(uint32_t(-1), max_logit, 0);
516 + max_logit = VLLM_SHFL_SYNC(max_logit, 0);
517
518 // Load rescaled exp sums to shared memory.
519 float* shared_exp_sums = reinterpret_cast<float*>(shared_mem + sizeof(float) * num_partitions);
@@ -539,9 +547,9 @@ __global__ void paged_attention_v2_reduce_kernel(
547 } // namespace vllm
548
549 #define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
542 - cudaFuncSetAttribute( \
543 - vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>, \
544 - cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem_size); \
550 + VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
551 + ((void*)vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>), \
552 + shared_mem_size); \
553 vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
554 <<<grid, block, shared_mem_size, stream>>>( \
555 out_ptr, \
csrc/attention/attention_utils.cuh
+2 -1
@@ -17,6 +17,7 @@
17 */
18 #pragma once
19
20 +#include "../cuda_compat.h"
21 #include "attention_dtypes.h"
22
23 #include <float.h>
@@ -39,7 +40,7 @@ inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
40 float qk = sum(qk_vec);
41 #pragma unroll
42 for (int mask = THREAD_GROUP_SIZE / 2; mask >= 1; mask /= 2) {
42 - qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
43 + qk += VLLM_SHFL_XOR_SYNC(qk, mask);
44 }
45 return qk;
46 }
csrc/attention/dtype_bfloat16.cuh
+16 -3
@@ -21,8 +21,17 @@
21 #include "attention_generic.cuh"
22 #include "dtype_float32.cuh"
23
24 -#include <cuda_bf16.h>
25 -#include <cuda_fp16.h>
24 +#ifndef USE_ROCM
25 + #include <cuda_bf16.h>
26 + #include <cuda_fp16.h>
27 +#else
28 + #include <hip/hip_bf16.h>
29 + #include <hip/hip_fp16.h>
30 +
31 + typedef __hip_bfloat162 __nv_bfloat162;
32 + typedef __hip_bfloat16 __nv_bfloat16;
33 +#endif
34 +
35 #include <stdint.h>
36
37 namespace vllm {
@@ -98,7 +107,11 @@ inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
107 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
108 assert(false);
109 #else
101 - return a + b;
110 + #ifndef USE_ROCM
111 + return a + b;
112 + #else
113 + return __hadd(a, b);
114 + #endif
115 #endif
116 }
117
csrc/attention/dtype_float16.cuh
+63 -5
@@ -21,6 +21,10 @@
21 #include "attention_generic.cuh"
22 #include "dtype_float32.cuh"
23
24 +#ifdef USE_ROCM
25 + #include <hip/hip_fp16.h>
26 +#endif
27 +
28 #include <stdint.h>
29
30 namespace vllm {
@@ -63,21 +67,47 @@ struct FloatVec<uint4> {
67
68 // Utility functions for type conversions.
69 inline __device__ uint32_t h0_h0(uint16_t a) {
70 +#ifndef USE_ROCM
71 uint32_t b;
72 asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
73 return b;
74 +#else
75 + union {
76 + uint32_t u32;
77 + uint16_t u16[2];
78 + } tmp;
79 + tmp.u16[0] = a;
80 + tmp.u16[1] = a;
81 + return tmp.u32;
82 +#endif
83 }
84
85 inline __device__ float half_to_float(uint16_t h) {
86 float f;
87 +#ifndef USE_ROCM
88 asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
89 +#else
90 + asm volatile("v_cvt_f32_f16 %0, %1;" : "=v"(f) : "v"(h));
91 +#endif
92 return f;
93 }
94
95 inline __device__ float2 half2_to_float2(uint32_t v) {
96 +#ifndef USE_ROCM
97 uint16_t lo, hi;
98 asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
99 return make_float2(half_to_float(lo), half_to_float(hi));
100 +#else
101 + union {
102 + uint32_t u32;
103 + uint16_t u16[2];
104 + } tmp;
105 + tmp.u32 = v;
106 + float2 ret;
107 + ret.x = half_to_float(tmp.u16[0]);
108 + ret.y = half_to_float(tmp.u16[1]);
109 + return ret;
110 +#endif
111 }
112
113 inline __device__ uint16_t float_to_half(float f) {
@@ -85,7 +115,11 @@ inline __device__ uint16_t float_to_half(float f) {
115 uint32_t u32;
116 uint16_t u16[2];
117 } tmp;
118 +#ifndef USE_ROCM
119 asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
120 +#else
121 + asm volatile("v_cvt_f16_f32 %0, %1;\n" : "=v"(tmp.u32) : "v"(f));
122 +#endif
123 return tmp.u16[0];
124 }
125
@@ -94,12 +128,16 @@ inline __device__ uint32_t float2_to_half2(float2 f) {
128 uint32_t u32;
129 uint16_t u16[2];
130 } tmp;
97 -
98 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
99 - asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
131 +#ifndef USE_ROCM
132 + #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
133 + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
134 + #else
135 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
136 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
137 + #endif
138 #else
101 - asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
102 - asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
139 + tmp.u16[0] = float_to_half(f.x);
140 + tmp.u16[1] = float_to_half(f.y);
141 #endif
142 return tmp.u32;
143 }
@@ -107,13 +145,21 @@ inline __device__ uint32_t float2_to_half2(float2 f) {
145 // Vector addition.
146 inline __device__ uint16_t add(uint16_t a, uint16_t b) {
147 uint16_t c;
148 +#ifndef USE_ROCM
149 asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
150 +#else
151 + asm volatile("v_add_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
152 +#endif
153 return c;
154 }
155
156 inline __device__ uint32_t add(uint32_t a, uint32_t b) {
157 uint32_t c;
158 +#ifndef USE_ROCM
159 asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
160 +#else
161 + asm volatile("v_pk_add_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
162 +#endif
163 return c;
164 }
165
@@ -158,14 +204,22 @@ inline __device__ Float8_ add(uint4 a, Float8_ fb) {
204 template<>
205 inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
206 uint16_t c;
207 +#ifndef USE_ROCM
208 asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
209 +#else
210 + asm volatile("v_mul_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
211 +#endif
212 return c;
213 }
214
215 template<>
216 inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
217 uint32_t c;
218 +#ifndef USE_ROCM
219 asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
220 +#else
221 + asm volatile("v_pk_mul_f16 %0, %1, %2;\n" : "=v"(c) : "v"(a), "v"(b));
222 +#endif
223 return c;
224 }
225
@@ -272,7 +326,11 @@ inline __device__ Float8_ mul(uint16_t a, uint4 b) {
326 // Vector fused multiply-add.
327 inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c) {
328 uint32_t d;
329 +#ifndef USE_ROCM
330 asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
331 +#else
332 + asm volatile("v_pk_fma_f16 %0, %1, %2, %3;\n" : "=v"(d) : "v"(a), "v"(b), "v"(c));
333 +#endif
334 return d;
335 }
336
csrc/cache_kernels.cu
+7 -6
@@ -1,6 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 +#include "cuda_compat.h"
5 #include "dispatch_utils.h"
6
7 #include <algorithm>
@@ -28,8 +29,8 @@ void swap_blocks(
29 TORCH_CHECK(false, "Invalid device combination");
30 }
31
31 - void *src_ptr = src.data_ptr();
32 - void *dst_ptr = dst.data_ptr();
32 + char *src_ptr = static_cast<char*>(src.data_ptr());
33 + char *dst_ptr = static_cast<char*>(dst.data_ptr());
34
35 const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
36 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
@@ -267,8 +268,8 @@ __global__ void gather_cached_kv_kernel(
268 + head_offset * block_size
269 + block_offset;
270
270 - key[tgt_key_idx] = __ldg(&key_cache[src_key_idx]);
271 - value[tgt_value_idx] = __ldg(&value_cache[src_value_idx]);
271 + key[tgt_key_idx] = VLLM_LDG(&key_cache[src_key_idx]);
272 + value[tgt_value_idx] = VLLM_LDG(&value_cache[src_value_idx]);
273 }
274 }
275
@@ -333,8 +334,8 @@ __global__ void gather_cached_kv_kernel_optimized(
334 src_key_indices[j] = src_key_idx;
335 src_value_indices[j] = src_value_idx;
336
336 - keys_to_store[j] = __ldg(&key_cache[src_key_idx]);
337 - values_to_store[j] = __ldg(&value_cache[src_value_idx]);
337 + keys_to_store[j] = VLLM_LDG(&key_cache[src_key_idx]);
338 + values_to_store[j] = VLLM_LDG(&value_cache[src_value_idx]);
339 }
340
341 #pragma unroll
csrc/cuda_compat.h new
+28
@@ -0,0 +1,28 @@
1 +#pragma once
2 +
3 +#ifndef USE_ROCM
4 + #define VLLM_LDG(arg) __ldg(arg)
5 +#else
6 + #define VLLM_LDG(arg) *(arg)
7 +#endif
8 +
9 +#ifndef USE_ROCM
10 + #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor_sync(uint32_t(-1), var, lane_mask)
11 +#else
12 + #define VLLM_SHFL_XOR_SYNC(var, lane_mask) __shfl_xor(var, lane_mask)
13 +#endif
14 +
15 +#ifndef USE_ROCM
16 + #define VLLM_SHFL_SYNC(var, src_lane) __shfl_sync(uint32_t(-1), var, src_lane)
17 +#else
18 + #define VLLM_SHFL_SYNC(var, src_lane) __shfl(var, src_lane)
19 +#endif
20 +
21 +#ifndef USE_ROCM
22 + #define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
23 + cudaFuncSetAttribute(FUNC, cudaFuncAttributeMaxDynamicSharedMemorySize, VAL)
24 +#else
25 + #define VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize(FUNC, VAL) \
26 + hipFuncSetAttribute(FUNC, hipFuncAttributeMaxDynamicSharedMemorySize, VAL)
27 +#endif
28 +
csrc/cuda_utils_kernels.cu
+3
@@ -1,3 +1,6 @@
1 +#ifdef USE_ROCM
2 + #include <hip/hip_runtime.h>
3 +#endif
4 int get_device_attribute(
5 int attribute,
6 int device_id)
csrc/ops.h
+2
@@ -61,12 +61,14 @@ void gelu_fast(
61 torch::Tensor& out,
62 torch::Tensor& input);
63
64 +#ifndef USE_ROCM
65 torch::Tensor awq_gemm(
66 torch::Tensor _in_feats,
67 torch::Tensor _kernel,
68 torch::Tensor _scaling_factors,
69 torch::Tensor _zeros,
70 int split_k_iters);
71 +#endif
72
73 void squeezellm_gemm(
74 torch::Tensor vec,
csrc/pos_encoding_kernels.cu
+5 -4
@@ -1,6 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 +#include "cuda_compat.h"
5 #include "dispatch_utils.h"
6
7 namespace vllm {
@@ -19,14 +20,14 @@ inline __device__ void apply_rotary_embedding(
20 // GPT-NeoX style rotary embedding.
21 x_index = rot_offset;
22 y_index = embed_dim + rot_offset;
22 - cos = __ldg(cos_ptr + x_index);
23 - sin = __ldg(sin_ptr + x_index);
23 + cos = VLLM_LDG(cos_ptr + x_index);
24 + sin = VLLM_LDG(sin_ptr + x_index);
25 } else {
26 // GPT-J style rotary embedding.
27 x_index = 2 * rot_offset;
28 y_index = 2 * rot_offset + 1;
28 - cos = __ldg(cos_ptr + x_index / 2);
29 - sin = __ldg(sin_ptr + x_index / 2);
29 + cos = VLLM_LDG(cos_ptr + x_index / 2);
30 + sin = VLLM_LDG(sin_ptr + x_index / 2);
31 }
32
33 const scalar_t x = arr[x_index];
csrc/pybind.cpp
+4
@@ -48,8 +48,12 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
48 &rotary_embedding,
49 "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
50
51 +#ifndef USE_ROCM
52 // Quantization ops
53 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
54 +#endif
55 +
56 +
57 ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
58
59 // Cache ops
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+74
@@ -20,9 +20,17 @@ __device__ inline unsigned int as_unsigned(int i) {
20
21 // 4-bit matvec kernel (LUT-based)
22 __global__ void NUQ4MatMulKernel(
23 +#ifndef USE_ROCM
24 const half2* __restrict__ vec,
25 +#else
26 + const __half2* __restrict__ vec,
27 +#endif
28 const int* __restrict__ mat,
29 +#ifndef USE_ROCM
30 half2* __restrict__ mul,
31 +#else
32 + float2* __restrict__ mul,
33 +#endif
34 const __half* __restrict__ lookup_table,
35 int height,
36 int width,
@@ -35,7 +43,11 @@ __global__ void NUQ4MatMulKernel(
43 int row = BLOCKHEIGHT4 * blockIdx.x;
44 int col = BLOCKWIDTH * blockIdx.y + threadIdx.x;
45
46 +#ifndef USE_ROCM
47 __shared__ half2 blockvec[blockwidth2];
48 +#else
49 + __shared__ __half2 blockvec[blockwidth2];
50 +#endif
51
52 __shared__ __half deq2[16][BLOCKWIDTH];
53 int off = threadIdx.x;
@@ -46,8 +58,13 @@ __global__ void NUQ4MatMulKernel(
58 }
59
60 __half res;
61 +#ifndef USE_ROCM
62 half2 res2;
63 half2 tmp2;
64 +#else
65 + __half2 res2;
66 + __half2 tmp2;
67 +#endif
68
69 int i;
70 int k;
@@ -68,48 +85,96 @@ __global__ void NUQ4MatMulKernel(
85 while (k < blockwidth2) {
86 tmp1 = as_unsigned(mat[i]);
87
88 +#ifndef USE_ROCM
89 res2 = {};
90 tmp2 = {};
91 +#else
92 + res2.x = __half_as_ushort(__float2half(0));
93 + res2.y = __half_as_ushort(__float2half(0));
94 + tmp2.x = __half_as_ushort(__float2half(0));
95 + tmp2.y = __half_as_ushort(__float2half(0));
96 +#endif
97
98 lut_index1 = tmp1 & 0xF;
99 lut_index2 = (tmp1 >> 4) & 0xF;
100 +#ifndef USE_ROCM
101 tmp2.x = deq2[lut_index1][off];
102 tmp2.y = deq2[lut_index2][off];
103 +#else
104 + tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
105 + tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
106 +#endif
107 res2 = __hfma2(tmp2, blockvec[k + 0], res2);
108
109 lut_index1 = (tmp1 >> 8) & 0xF;
110 lut_index2 = (tmp1 >> 12) & 0xF;
111 +#ifndef USE_ROCM
112 tmp2.x = deq2[lut_index1][off];
113 tmp2.y = deq2[lut_index2][off];
114 +#else
115 + tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
116 + tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
117 +#endif
118 res2 = __hfma2(tmp2, blockvec[k + 1], res2);
119
120 lut_index1 = (tmp1 >> 16) & 0xF;
121 lut_index2 = (tmp1 >> 20) & 0xF;
122 +#ifndef USE_ROCM
123 tmp2.x = deq2[lut_index1][off];
124 tmp2.y = deq2[lut_index2][off];
125 +#else
126 + tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
127 + tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
128 +#endif
129 res2 = __hfma2(tmp2, blockvec[k + 2], res2);
130
131 lut_index1 = (tmp1 >> 24) & 0xF;
132 lut_index2 = (tmp1 >> 28) & 0xF;
133 +#ifndef USE_ROCM
134 tmp2.x = deq2[lut_index1][off];
135 tmp2.y = deq2[lut_index2][off];
136 +#else
137 + tmp2.x = __half_as_ushort(deq2[lut_index1][off]);
138 + tmp2.y = __half_as_ushort(deq2[lut_index2][off]);
139 +#endif
140 res2 = __hfma2(tmp2, blockvec[k + 3], res2);
141
142 +#ifndef USE_ROCM
143 res = __hadd(__hadd(res2.x, res2.y), res);
144 +#else
145 + res = __hadd(__hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y)), res);
146 +#endif
147
148 i += width;
149 k += 4;
150 }
151
152 // col%2 -> only set one of the two values
153 +#ifndef USE_ROCM
154 half2 res3 = {};
155 if (col % 2 == 0) {
156 res3.x = res;
157 } else {
158 res3.y = res;
159 }
160 +#else
161 + __half2 res3;
162 + res3.x = __half_as_ushort(__float2half(0));
163 + res3.y = __half_as_ushort(__float2half(0));
164 + if (col % 2 == 0) {
165 + res3.x = __half_as_ushort(res);
166 + } else {
167 + res3.y = __half_as_ushort(res);
168 + }
169 +#endif
170
171 +#ifndef USE_ROCM
172 atomicAdd(&mul[b * width / 2 + col / 2], res3);
173 +#else
174 + int tmp_addr = b * width / 2 + col / 2;
175 + atomicAdd(&(mul[tmp_addr].x), __half2float(__ushort_as_half(res3.x)));
176 + atomicAdd(&(mul[tmp_addr].y), __half2float(__ushort_as_half(res3.y)));
177 +#endif
178 }
179 }
180
@@ -136,10 +201,19 @@ void squeezellm_gemm(
201 dim3 threads(BLOCKWIDTH);
202
203 vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
204 +#ifndef USE_ROCM
205 (half2*) vec.data<at::Half>(),
206 +#else
207 + (__half2*) vec.data_ptr<at::Half>(),
208 +#endif
209 mat.data_ptr<int>(),
210 +#ifndef USE_ROCM
211 (half2*) mul.data<at::Half>(),
212 (__half*) lookup_table.data<at::Half>(),
213 +#else
214 + (float2*) mul.data_ptr<float>(),
215 + (__half*) lookup_table.data_ptr<at::Half>(),
216 +#endif
217 height, width, batch, vec_height
218 );
219 }
csrc/reduction_utils.cuh
+3 -1
@@ -17,13 +17,15 @@
17 */
18 #pragma once
19
20 +#include "cuda_compat.h"
21 +
22 namespace vllm {
23
24 template<typename T>
25 __inline__ __device__ T warpReduceSum(T val) {
26 #pragma unroll
27 for (int mask = 16; mask > 0; mask >>= 1)
26 - val += __shfl_xor_sync(0xffffffff, val, mask, 32);
28 + val += VLLM_SHFL_XOR_SYNC(val, mask);
29 return val;
30 }
31
docs/source/getting_started/amd-installation.rst new
+143
@@ -0,0 +1,143 @@
1 +.. _installation_rocm:
2 +
3 +Installation with ROCm
4 +======================
5 +
6 +vLLM 0.2.x onwards supports model inferencing and serving on AMD GPUs with ROCm.
7 +At the moment AWQ quantization is not supported in ROCm, but SqueezeLLM quantization has been ported.
8 +Data types currently supported in ROCm are FP16 and BF16.
9 +
10 +Requirements
11 +------------
12 +
13 +* OS: Linux
14 +* Python: 3.8 -- 3.11 (Verified on 3.10)
15 +* GPU: MI200s
16 +* Pytorch 2.0.1/2.1.1/2.2
17 +* ROCm 5.7
18 +
19 +Installation options:
20 +
21 +#. :ref:`(Recommended) Quick start with vLLM pre-installed in Docker Image <quick_start_docker_rocm>`
22 +#. :ref:`Build from source <build_from_source_rocm>`
23 +#. :ref:`Build from source with docker <build_from_source_docker_rocm>`
24 +
25 +.. _quick_start_docker_rocm:
26 +
27 +(Recommended) Option 1: Quick start with vLLM pre-installed in Docker Image
28 +---------------------------------------------------------------------------
29 +
30 +.. code-block:: console
31 +
32 + $ docker pull embeddedllminfo/vllm-rocm:vllm-v0.2.3
33 + $ docker run -it \
34 + --network=host \
35 + --group-add=video \
36 + --ipc=host \
37 + --cap-add=SYS_PTRACE \
38 + --security-opt seccomp=unconfined \
39 + --device /dev/kfd \
40 + --device /dev/dri \
41 + -v <path/to/model>:/app/model \
42 + embeddedllminfo/vllm-rocm \
43 + bash
44 +
45 +
46 +.. _build_from_source_rocm:
47 +
48 +Option 2: Build from source
49 +---------------------------
50 +
51 +You can build and install vLLM from source:
52 +
53 +0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
54 +
55 +- `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
56 +- `Pytorch <https://pytorch.org/>`_
57 +
58 + .. code-block:: console
59 +
60 + $ pip install torch==2.2.0.dev20231206+rocm5.7 --index-url https://download.pytorch.org/whl/nightly/rocm5.7 # tested version
61 +
62 +
63 +1. Install `flash attention for ROCm <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm>`_
64 +
65 + Install ROCm's flash attention (v2.0.4) following the instructions from `ROCmSoftwarePlatform/flash-attention <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm#amd-gpurocm-support>`_
66 +
67 +.. note::
68 + - If you are using rocm5.7 with pytorch 2.1.0 onwards, you don't need to apply the `hipify_python.patch`. You can build the ROCm flash attention directly.
69 + - If you fail to install `ROCmSoftwarePlatform/flash-attention`, try cloning from the commit `6fd2f8e572805681cd67ef8596c7e2ce521ed3c6`.
70 + - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
71 + - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
72 +
73 +2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
74 +
75 + .. code-block:: console
76 +
77 + $ pip install xformers==0.0.22.post7 --no-deps
78 + $ bash patch_xformers-0.0.22.post7.rocm.sh
79 +
80 +3. Build vLLM.
81 +
82 + .. code-block:: console
83 +
84 + $ cd vllm
85 + $ pip install -U -r requirements-rocm.txt
86 + $ python setup.py install # This may take 5-10 minutes. Currently, `pip install .`` does not work for ROCm installation
87 +
88 +
89 +.. _build_from_source_docker_rocm:
90 +
91 +Option 3: Build from source with docker
92 +-----------------------------------------------------
93 +
94 +You can build and install vLLM from source:
95 +
96 +Build a docker image from `Dockerfile.rocm`, and launch a docker container.
97 +
98 +.. code-block:: console
99 +
100 + $ docker build -f Dockerfile.rocm -t vllm-rocm .
101 + $ docker run -it \
102 + --network=host \
103 + --group-add=video \
104 + --ipc=host \
105 + --cap-add=SYS_PTRACE \
106 + --security-opt seccomp=unconfined \
107 + --device /dev/kfd \
108 + --device /dev/dri \
109 + -v <path/to/model>:/app/model \
110 + vllm-rocm \
111 + bash
112 +
113 +Alternatively, if you plan to install vLLM-ROCm on a local machine or start from a fresh docker image (e.g. rocm/pytorch), you can follow the steps below:
114 +
115 +0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
116 +
117 +- `ROCm <https://rocm.docs.amd.com/en/latest/deploy/linux/index.html>`_
118 +- `Pytorch <https://pytorch.org/>`_
119 +
120 +1. Install `flash attention for ROCm <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm>`_
121 +
122 + Install ROCm's flash attention (v2.0.4) following the instructions from `ROCmSoftwarePlatform/flash-attention <https://github.com/ROCmSoftwarePlatform/flash-attention/tree/flash_attention_for_rocm#amd-gpurocm-support>`_
123 +
124 +.. note::
125 + - If you are using rocm5.7 with pytorch 2.1.0 onwards, you don't need to apply the `hipify_python.patch`. You can build the ROCm flash attention directly.
126 + - If you fail to install `ROCmSoftwarePlatform/flash-attention`, try cloning from the commit `6fd2f8e572805681cd67ef8596c7e2ce521ed3c6`.
127 + - ROCm's Flash-attention-2 (v2.0.4) does not support sliding windows attention.
128 + - You might need to downgrade the "ninja" version to 1.10 it is not used when compiling flash-attention-2 (e.g. `pip install ninja==1.10.2.4`)
129 +
130 +2. Setup `xformers==0.0.22.post7` without dependencies, and apply patches to adapt for ROCm flash attention
131 +
132 + .. code-block:: console
133 +
134 + $ pip install xformers==0.0.22.post7 --no-deps
135 + $ bash patch_xformers-0.0.22.post7.rocm.sh
136 +
137 +3. Build vLLM.
138 +
139 + .. code-block:: console
140 +
141 + $ cd vllm
142 + $ pip install -U -r requirements-rocm.txt
143 + $ python setup.py install # This may take 5-10 minutes.
docs/source/index.rst
+2
@@ -39,6 +39,7 @@ vLLM is flexible and easy to use with:
39 * Tensor parallelism support for distributed inference
40 * Streaming outputs
41 * OpenAI-compatible API server
42 +* Support NVIDIA CUDA and AMD ROCm.
43
44 For more information, check out the following:
45
@@ -56,6 +57,7 @@ Documentation
57 :caption: Getting Started
58
59 getting_started/installation
60 + getting_started/amd-installation
61 getting_started/quickstart
62
63 .. toctree::
patch_xformers-0.0.22.post7.rocm.sh new
+22
@@ -0,0 +1,22 @@
1 +#!/bin/bash
2 +export XFORMERS_FMHA_FLASH_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.flash.__file__)')
3 +export XFORMERS_FMHA_COMMON_PATH=$(python -c 'from xformers import ops as xops; print(xops.fmha.common.__file__)')
4 +
5 +echo $XFORMERS_FMHA_FLASH_PATH
6 +echo $XFORMERS_FMHA_COMMON_PATH
7 +
8 +if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"; then
9 + echo "Applying patch to ${XFORMERS_FMHA_FLASH_PATH}"
10 + patch -p0 $XFORMERS_FMHA_FLASH_PATH "./rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch"
11 + echo "Successfully patch ${XFORMERS_FMHA_FLASH_PATH}"
12 +else
13 + echo "${XFORMERS_FMHA_FLASH_PATH} was patched before"
14 +fi
15 +
16 +if ! patch -R -p0 -s -f --dry-run $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"; then
17 + echo "Applying patch to ${XFORMERS_FMHA_COMMON_PATH}"
18 + patch -p0 $XFORMERS_FMHA_COMMON_PATH "./rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch"
19 + echo "Successfully patch ${XFORMERS_FMHA_COMMON_PATH}"
20 +else
21 + echo "${XFORMERS_FMHA_COMMON_PATH} was patched before"
22 +fi
requirements-rocm.txt new
+17
@@ -0,0 +1,17 @@
1 +ninja # For faster builds.
2 +typing-extensions>=4.8.0
3 +starlette
4 +psutil
5 +ray >= 2.5.1
6 +pandas # Required for Ray data.
7 +pyarrow # Required for Ray data.
8 +sentencepiece # Required for LLaMA tokenizer.
9 +numpy
10 +tokenizers>=0.15.0
11 +huggingface_hub<0.18,>=0.16.4
12 +einops # Required for phi-1_5
13 +transformers >= 4.34.0 # Required for Mistral.
14 +fastapi
15 +uvicorn[standard]
16 +pydantic == 1.10.13 # Required for OpenAI server.
17 +aioprometheus[starlette]
rocm_patch/commonpy_xformers-0.0.22.post7.rocm.patch new
+13
@@ -0,0 +1,13 @@
1 +--- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/common.py 2023-11-29 03:17:03.930103539 +0000
2 ++++ common.py 2023-11-28 16:14:19.846233146 +0000
3 +@@ -298,8 +298,8 @@
4 + dtype = d.query.dtype
5 + if device_type not in cls.SUPPORTED_DEVICES:
6 + reasons.append(f"device={device_type} (supported: {cls.SUPPORTED_DEVICES})")
7 +- if device_type == "cuda" and not _built_with_cuda:
8 +- reasons.append("xFormers wasn't build with CUDA support")
9 ++ #if device_type == "cuda" and not _built_with_cuda:
10 ++ # reasons.append("xFormers wasn't build with CUDA support")
11 + if device_type == "cuda":
12 + device_capability = torch.cuda.get_device_capability(d.device)
13 + if device_capability < cls.CUDA_MINIMUM_COMPUTE_CAPABILITY:
rocm_patch/flashpy_xformers-0.0.22.post7.rocm.patch new
+134
@@ -0,0 +1,134 @@
1 +--- /opt/conda/envs/py_3.10/lib/python3.10/site-packages/xformers/ops/fmha/flash.py 2023-11-29 03:17:03.930103539 +0000
2 ++++ flash.py 2023-11-28 16:14:25.206128903 +0000
3 +@@ -31,39 +31,39 @@
4 +
5 + FLASH_VERSION = "0.0.0"
6 + try:
7 +- try:
8 +- from ... import _C_flashattention # type: ignore[attr-defined]
9 +- from ..._cpp_lib import _build_metadata
10 +-
11 +- if _build_metadata is not None:
12 +- FLASH_VERSION = _build_metadata.flash_version
13 +- except ImportError:
14 +- import flash_attn
15 +- from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
16 +-
17 +- FLASH_VERSION = flash_attn.__version__
18 +- flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
19 +- if flash_ver_parsed < (2, 3):
20 +- raise ImportError("Requires 2.3 for sliding window support")
21 ++ #try:
22 ++ # from ... import _C_flashattention # type: ignore[attr-defined]
23 ++ # from ..._cpp_lib import _build_metadata
24 ++
25 ++ # if _build_metadata is not None:
26 ++ # FLASH_VERSION = _build_metadata.flash_version
27 ++ #except ImportError:
28 ++ import flash_attn
29 ++ from flash_attn.flash_attn_interface import flash_attn_cuda as _C_flashattention
30 ++
31 ++ FLASH_VERSION = flash_attn.__version__
32 ++ # flash_ver_parsed = tuple(int(s) for s in FLASH_VERSION.split(".")[:2])
33 ++ # if flash_ver_parsed < (2, 3):
34 ++ # raise ImportError("Requires 2.3 for sliding window support")
35 +
36 + # create library so that flash-attn goes through the PyTorch Dispatcher
37 +- _flash_lib = torch.library.Library("xformers_flash", "DEF")
38 ++ #_flash_lib = torch.library.Library("xformers_flash", "DEF")
39 +
40 +- _flash_lib.define(
41 +- "flash_fwd(Tensor query, Tensor key, Tensor value, "
42 +- "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
43 +- "int max_seqlen_q, int max_seqlen_k, "
44 +- "float p, float softmax_scale, "
45 +- "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
46 +- )
47 +-
48 +- _flash_lib.define(
49 +- "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
50 +- "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
51 +- "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
52 +- "int max_seqlen_q, int max_seqlen_k, "
53 +- "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
54 +- )
55 ++ #_flash_lib.define(
56 ++ # "flash_fwd(Tensor query, Tensor key, Tensor value, "
57 ++ # "Tensor? cu_seqlens_q, Tensor? cu_seqlens_k, "
58 ++ # "int max_seqlen_q, int max_seqlen_k, "
59 ++ # "float p, float softmax_scale, "
60 ++ # "bool is_causal, int window_size, bool return_softmax) -> (Tensor, Tensor, Tensor)"
61 ++ #)
62 ++
63 ++ #_flash_lib.define(
64 ++ # "flash_bwd(Tensor dout, Tensor query, Tensor key, Tensor value, "
65 ++ # "Tensor out, Tensor softmax_lse_, Tensor dq, Tensor dk, Tensor dv, "
66 ++ # "Tensor cu_seqlens_q, Tensor cu_seqlens_k, "
67 ++ # "int max_seqlen_q, int max_seqlen_k, "
68 ++ # "float p, float softmax_scale, bool is_causal, int window_size, Tensor rng_state) -> (Tensor, Tensor, Tensor)"
69 ++ #)
70 +
71 + def _flash_fwd(
72 + query,
73 +@@ -98,8 +98,8 @@
74 + p,
75 + softmax_scale,
76 + is_causal,
77 +- window_size - 1, # window_size_left
78 +- -1, # window_size_right
79 ++ # window_size - 1, # window_size_left
80 ++ # -1, # window_size_right
81 + return_softmax,
82 + None, # rng
83 + )
84 +@@ -127,8 +127,8 @@
85 + softmax_scale,
86 + False,
87 + is_causal,
88 +- window_size - 1, # window_size_left
89 +- -1, # window_size_right
90 ++ # window_size - 1, # window_size_left
91 ++ # -1, # window_size_right
92 + return_softmax,
93 + None,
94 + )
95 +@@ -169,8 +169,8 @@
96 + p,
97 + softmax_scale,
98 + is_causal,
99 +- window_size - 1, # window_size_left
100 +- -1, # window_size_right
101 ++ # window_size - 1, # window_size_left
102 ++ # -1, # window_size_right
103 + None,
104 + rng_state,
105 + )
106 +@@ -193,15 +193,15 @@
107 + softmax_scale,
108 + False, # zero_tensors
109 + is_causal,
110 +- window_size - 1, # window_size_left
111 +- -1, # window_size_right
112 ++ # window_size - 1, # window_size_left
113 ++ # -1, # window_size_right
114 + None,
115 + rng_state,
116 + )
117 + return dq, dk, dv
118 +
119 +- _flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
120 +- _flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
121 ++ #_flash_lib.impl("flash_fwd", _flash_fwd, "CUDA")
122 ++ #_flash_lib.impl("flash_bwd", _flash_bwd, "CUDA")
123 + except ImportError:
124 + pass
125 +
126 +@@ -348,7 +348,7 @@
127 + implementation.
128 + """
129 +
130 +- OPERATOR = get_operator("xformers_flash", "flash_fwd")
131 ++ OPERATOR = _flash_fwd # get_operator("xformers_flash", "flash_fwd")
132 + SUPPORTED_DEVICES: Set[str] = {"cuda"}
133 + CUDA_MINIMUM_COMPUTE_CAPABILITY = (8, 0)
134 + SUPPORTED_DTYPES: Set[torch.dtype] = {torch.half, torch.bfloat16}
setup.py
+159 -73
@@ -8,27 +8,83 @@ import warnings
8 from packaging.version import parse, Version
9 import setuptools
10 import torch
11 -from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME
11 +from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
12
13 ROOT_DIR = os.path.dirname(__file__)
14
15 MAIN_CUDA_VERSION = "12.1"
16
17 # Supported NVIDIA GPU architectures.
18 -SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
18 +NVIDIA_SUPPORTED_ARCHS = {"7.0", "7.5", "8.0", "8.6", "8.9", "9.0"}
19 +ROCM_SUPPORTED_ARCHS = {"gfx90a", "gfx908", "gfx906", "gfx1030", "gfx1100"}
20 +# SUPPORTED_ARCHS = NVIDIA_SUPPORTED_ARCHS.union(ROCM_SUPPORTED_ARCHS)
21 +
22 +
23 +def _is_hip() -> bool:
24 + return torch.version.hip is not None
25 +
26 +
27 +def _is_cuda() -> bool:
28 + return torch.version.cuda is not None
29 +
30
31 # Compiler flags.
32 CXX_FLAGS = ["-g", "-O2", "-std=c++17"]
33 # TODO(woosuk): Should we use -O3?
34 NVCC_FLAGS = ["-O2", "-std=c++17"]
35
36 +if _is_hip():
37 + if ROCM_HOME is None:
38 + raise RuntimeError(
39 + "Cannot find ROCM_HOME. ROCm must be available to build the package."
40 + )
41 + NVCC_FLAGS += ["-DUSE_ROCM"]
42 +
43 +if _is_cuda() and CUDA_HOME is None:
44 + raise RuntimeError(
45 + "Cannot find CUDA_HOME. CUDA must be available to build the package.")
46 +
47 ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
48 CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
49 NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
50
29 -if CUDA_HOME is None:
30 - raise RuntimeError(
31 - "Cannot find CUDA_HOME. CUDA must be available to build the package.")
51 +
52 +def get_amdgpu_offload_arch():
53 + command = "/opt/rocm/llvm/bin/amdgpu-offload-arch"
54 + try:
55 + output = subprocess.check_output([command])
56 + return output.decode('utf-8').strip()
57 + except subprocess.CalledProcessError as e:
58 + error_message = f"Error: {e}"
59 + raise RuntimeError(error_message) from e
60 + except FileNotFoundError as e:
61 + # If the command is not found, print an error message
62 + error_message = f"The command {command} was not found."
63 + raise RuntimeError(error_message) from e
64 +
65 + return None
66 +
67 +
68 +def get_hipcc_rocm_version():
69 + # Run the hipcc --version command
70 + result = subprocess.run(['hipcc', '--version'],
71 + stdout=subprocess.PIPE,
72 + stderr=subprocess.STDOUT,
73 + text=True)
74 +
75 + # Check if the command was executed successfully
76 + if result.returncode != 0:
77 + print("Error running 'hipcc --version'")
78 + return None
79 +
80 + # Extract the version using a regular expression
81 + match = re.search(r'HIP version: (\S+)', result.stdout)
82 + if match:
83 + # Return the version string
84 + return match.group(1)
85 + else:
86 + print("Could not find HIP version in the output")
87 + return None
88
89
90 def get_nvcc_cuda_version(cuda_dir: str) -> Version:
@@ -61,20 +117,22 @@ def get_torch_arch_list() -> Set[str]:
117 return set()
118
119 # Filter out the invalid architectures and print a warning.
64 - valid_archs = SUPPORTED_ARCHS.union({s + "+PTX" for s in SUPPORTED_ARCHS})
120 + valid_archs = NVIDIA_SUPPORTED_ARCHS.union(
121 + {s + "+PTX"
122 + for s in NVIDIA_SUPPORTED_ARCHS})
123 arch_list = torch_arch_list.intersection(valid_archs)
124 # If none of the specified architectures are valid, raise an error.
125 if not arch_list:
126 raise RuntimeError(
69 - "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
127 + "None of the CUDA/ROCM architectures in `TORCH_CUDA_ARCH_LIST` env "
128 f"variable ({env_arch_list}) is supported. "
71 - f"Supported CUDA architectures are: {valid_archs}.")
129 + f"Supported CUDA/ROCM architectures are: {valid_archs}.")
130 invalid_arch_list = torch_arch_list - valid_archs
131 if invalid_arch_list:
132 warnings.warn(
75 - f"Unsupported CUDA architectures ({invalid_arch_list}) are "
133 + f"Unsupported CUDA/ROCM architectures ({invalid_arch_list}) are "
134 "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
77 - f"({env_arch_list}). Supported CUDA architectures are: "
135 + f"({env_arch_list}). Supported CUDA/ROCM architectures are: "
136 f"{valid_archs}.",
137 stacklevel=2)
138 return arch_list
@@ -82,7 +140,7 @@ def get_torch_arch_list() -> Set[str]:
140
141 # First, check the TORCH_CUDA_ARCH_LIST environment variable.
142 compute_capabilities = get_torch_arch_list()
85 -if not compute_capabilities:
143 +if _is_cuda() and not compute_capabilities:
144 # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
145 # GPUs on the current machine.
146 device_count = torch.cuda.device_count()
@@ -93,69 +151,84 @@ if not compute_capabilities:
151 "GPUs with compute capability below 7.0 are not supported.")
152 compute_capabilities.add(f"{major}.{minor}")
153
96 -nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
97 -if not compute_capabilities:
98 - # If no GPU is specified nor available, add all supported architectures
99 - # based on the NVCC CUDA version.
100 - compute_capabilities = SUPPORTED_ARCHS.copy()
101 - if nvcc_cuda_version < Version("11.1"):
102 - compute_capabilities.remove("8.6")
154 +if _is_cuda():
155 + nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
156 + if not compute_capabilities:
157 + # If no GPU is specified nor available, add all supported architectures
158 + # based on the NVCC CUDA version.
159 + compute_capabilities = NVIDIA_SUPPORTED_ARCHS.copy()
160 + if nvcc_cuda_version < Version("11.1"):
161 + compute_capabilities.remove("8.6")
162 + if nvcc_cuda_version < Version("11.8"):
163 + compute_capabilities.remove("8.9")
164 + compute_capabilities.remove("9.0")
165 + # Validate the NVCC CUDA version.
166 + if nvcc_cuda_version < Version("11.0"):
167 + raise RuntimeError(
168 + "CUDA 11.0 or higher is required to build the package.")
169 + if (nvcc_cuda_version < Version("11.1")
170 + and any(cc.startswith("8.6") for cc in compute_capabilities)):
171 + raise RuntimeError(
172 + "CUDA 11.1 or higher is required for compute capability 8.6.")
173 if nvcc_cuda_version < Version("11.8"):
104 - compute_capabilities.remove("8.9")
105 - compute_capabilities.remove("9.0")
106 -
107 -# Validate the NVCC CUDA version.
108 -if nvcc_cuda_version < Version("11.0"):
109 - raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
110 -if (nvcc_cuda_version < Version("11.1")
111 - and any(cc.startswith("8.6") for cc in compute_capabilities)):
112 - raise RuntimeError(
113 - "CUDA 11.1 or higher is required for compute capability 8.6.")
114 -if nvcc_cuda_version < Version("11.8"):
115 - if any(cc.startswith("8.9") for cc in compute_capabilities):
116 - # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
117 - # However, GPUs with compute capability 8.9 can also run the code generated by
118 - # the previous versions of CUDA 11 and targeting compute capability 8.0.
119 - # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
120 - # instead of 8.9.
121 - warnings.warn(
122 - "CUDA 11.8 or higher is required for compute capability 8.9. "
123 - "Targeting compute capability 8.0 instead.",
124 - stacklevel=2)
125 - compute_capabilities = set(cc for cc in compute_capabilities
126 - if not cc.startswith("8.9"))
127 - compute_capabilities.add("8.0+PTX")
128 - if any(cc.startswith("9.0") for cc in compute_capabilities):
174 + if any(cc.startswith("8.9") for cc in compute_capabilities):
175 + # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
176 + # However, GPUs with compute capability 8.9 can also run the code generated by
177 + # the previous versions of CUDA 11 and targeting compute capability 8.0.
178 + # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
179 + # instead of 8.9.
180 + warnings.warn(
181 + "CUDA 11.8 or higher is required for compute capability 8.9. "
182 + "Targeting compute capability 8.0 instead.",
183 + stacklevel=2)
184 + compute_capabilities = set(cc for cc in compute_capabilities
185 + if not cc.startswith("8.9"))
186 + compute_capabilities.add("8.0+PTX")
187 + if any(cc.startswith("9.0") for cc in compute_capabilities):
188 + raise RuntimeError(
189 + "CUDA 11.8 or higher is required for compute capability 9.0.")
190 +
191 + # Add target compute capabilities to NVCC flags.
192 + for capability in compute_capabilities:
193 + num = capability[0] + capability[2]
194 + NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
195 + if capability.endswith("+PTX"):
196 + NVCC_FLAGS += [
197 + "-gencode", f"arch=compute_{num},code=compute_{num}"
198 + ]
199 +
200 + # Use NVCC threads to parallelize the build.
201 + if nvcc_cuda_version >= Version("11.2"):
202 + nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
203 + num_threads = min(os.cpu_count(), nvcc_threads)
204 + NVCC_FLAGS += ["--threads", str(num_threads)]
205 +
206 +elif _is_hip():
207 + amd_arch = get_amdgpu_offload_arch()
208 + if amd_arch not in ROCM_SUPPORTED_ARCHS:
209 raise RuntimeError(
130 - "CUDA 11.8 or higher is required for compute capability 9.0.")
210 + f"Only the following arch is supported: {ROCM_SUPPORTED_ARCHS}"
211 + f"amdgpu_arch_found: {amd_arch}")
212 +
213 +ext_modules = []
214
132 -# Add target compute capabilities to NVCC flags.
133 -for capability in compute_capabilities:
134 - num = capability[0] + capability[2]
135 - NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
136 - if capability.endswith("+PTX"):
137 - NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=compute_{num}"]
215 +vllm_extension_sources = [
216 + "csrc/cache_kernels.cu",
217 + "csrc/attention/attention_kernels.cu",
218 + "csrc/pos_encoding_kernels.cu",
219 + "csrc/activation_kernels.cu",
220 + "csrc/layernorm_kernels.cu",
221 + "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
222 + "csrc/cuda_utils_kernels.cu",
223 + "csrc/pybind.cpp",
224 +]
225
139 -# Use NVCC threads to parallelize the build.
140 -if nvcc_cuda_version >= Version("11.2"):
141 - nvcc_threads = int(os.getenv("NVCC_THREADS", 8))
142 - num_threads = min(os.cpu_count(), nvcc_threads)
143 - NVCC_FLAGS += ["--threads", str(num_threads)]
226 +if _is_cuda():
227 + vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
228
145 -ext_modules = []
229 vllm_extension = CUDAExtension(
230 name="vllm._C",
148 - sources=[
149 - "csrc/cache_kernels.cu",
150 - "csrc/attention/attention_kernels.cu",
151 - "csrc/pos_encoding_kernels.cu",
152 - "csrc/activation_kernels.cu",
153 - "csrc/layernorm_kernels.cu",
154 - "csrc/quantization/awq/gemm_kernels.cu",
155 - "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
156 - "csrc/cuda_utils_kernels.cu",
157 - "csrc/pybind.cpp",
158 - ],
231 + sources=vllm_extension_sources,
232 extra_compile_args={
233 "cxx": CXX_FLAGS,
234 "nvcc": NVCC_FLAGS,
@@ -183,10 +256,19 @@ def find_version(filepath: str) -> str:
256
257 def get_vllm_version() -> str:
258 version = find_version(get_path("vllm", "__init__.py"))
186 - cuda_version = str(nvcc_cuda_version)
187 - if cuda_version != MAIN_CUDA_VERSION:
188 - cuda_version_str = cuda_version.replace(".", "")[:3]
189 - version += f"+cu{cuda_version_str}"
259 +
260 + if _is_hip():
261 + # Get the HIP version
262 + hipcc_version = get_hipcc_rocm_version()
263 + if hipcc_version != MAIN_CUDA_VERSION:
264 + rocm_version_str = hipcc_version.replace(".", "")[:3]
265 + version += f"+rocm{rocm_version_str}"
266 + else:
267 + cuda_version = str(nvcc_cuda_version)
268 + if cuda_version != MAIN_CUDA_VERSION:
269 + cuda_version_str = cuda_version.replace(".", "")[:3]
270 + version += f"+cu{cuda_version_str}"
271 +
272 return version
273
274
@@ -201,8 +283,12 @@ def read_readme() -> str:
283
284 def get_requirements() -> List[str]:
285 """Get Python package dependencies from requirements.txt."""
204 - with open(get_path("requirements.txt")) as f:
205 - requirements = f.read().strip().split("\n")
286 + if _is_hip():
287 + with open(get_path("requirements-rocm.txt")) as f:
288 + requirements = f.read().strip().split("\n")
289 + else:
290 + with open(get_path("requirements.txt")) as f:
291 + requirements = f.read().strip().split("\n")
292 return requirements
293
294
vllm/config.py
+35 -4
@@ -6,7 +6,7 @@ from transformers import PretrainedConfig
6
7 from vllm.logger import init_logger
8 from vllm.transformers_utils.config import get_config
9 -from vllm.utils import get_cpu_memory
9 +from vllm.utils import get_cpu_memory, is_hip
10
11 logger = init_logger(__name__)
12
@@ -98,12 +98,27 @@ class ModelConfig:
98
99 def _verify_load_format(self) -> None:
100 load_format = self.load_format.lower()
101 - if load_format not in [
102 - "auto", "pt", "safetensors", "npcache", "dummy"
103 - ]:
101 + supported_load_format = [
102 + "auto", "pt", "safetensors", "npcache", "dummy"
103 + ]
104 + rocm_not_supported_load_format = ["safetensors"]
105 + if load_format not in supported_load_format:
106 raise ValueError(
107 f"Unknown load format: {self.load_format}. Must be one of "
108 "'auto', 'pt', 'safetensors', 'npcache', or 'dummy'.")
109 + if is_hip():
110 + if load_format in ["safetensors"]:
111 + rocm_supported_load_format = [
112 + f for f in supported_load_format
113 + if (f not in rocm_not_supported_load_format)
114 + ]
115 + raise ValueError(
116 + f"load format \'{load_format}\' is not supported in ROCm. "
117 + f"Supported load format are "
118 + f"{rocm_supported_load_format}")
119 + # Force ROCm to load from pt weights if nothing specific is set
120 + if load_format == "auto":
121 + load_format = "pt"
122 self.load_format = load_format
123
124 def _verify_tokenizer_mode(self) -> None:
@@ -116,6 +131,7 @@ class ModelConfig:
131
132 def _verify_quantization(self) -> None:
133 supported_quantization = ["awq", "squeezellm"]
134 + rocm_not_supported_quantization = ["awq"]
135 if self.quantization is not None:
136 self.quantization = self.quantization.lower()
137
@@ -137,6 +153,11 @@ class ModelConfig:
153 raise ValueError(
154 f"Unknown quantization method: {self.quantization}. Must "
155 f"be one of {supported_quantization}.")
156 + if is_hip(
157 + ) and self.quantization in rocm_not_supported_quantization:
158 + raise ValueError(
159 + f"{self.quantization} quantization is currently not supported "
160 + f"in ROCm.")
161 logger.warning(f"{self.quantization} quantization is not fully "
162 "optimized yet. The speed can be slower than "
163 "non-quantized models.")
@@ -364,6 +385,8 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
385 "bfloat16": torch.bfloat16,
386 }
387
388 +_ROCM_NOT_SUPPORTED_DTYPE = ["float", "float32"]
389 +
390
391 def _get_and_verify_dtype(
392 config: PretrainedConfig,
@@ -393,6 +416,14 @@ def _get_and_verify_dtype(
416 else:
417 raise ValueError(f"Unknown dtype: {dtype}")
418
419 + if is_hip() and torch_dtype == torch.float32:
420 + rocm_supported_dtypes = [
421 + k for k, v in _STR_DTYPE_TO_TORCH_DTYPE.items()
422 + if (k not in _ROCM_NOT_SUPPORTED_DTYPE)
423 + ]
424 + raise ValueError(f"dtype \'{dtype}\' is not supported in ROCm. "
425 + f"Supported dtypes are {rocm_supported_dtypes}")
426 +
427 # Verify the dtype.
428 if torch_dtype != config_dtype:
429 if torch_dtype == torch.float32:
vllm/engine/ray_utils.py
+7 -1
@@ -3,6 +3,7 @@ from typing import Optional, Tuple, TYPE_CHECKING
3
4 from vllm.config import ParallelConfig
5 from vllm.logger import init_logger
6 +from vllm.utils import is_hip
7
8 logger = init_logger(__name__)
9
@@ -73,7 +74,12 @@ def initialize_cluster(
74 "Ray is not installed. Please install Ray to use distributed "
75 "serving.")
76 # Connect to a ray cluster.
76 - ray.init(address=ray_address, ignore_reinit_error=True)
77 + if is_hip():
78 + ray.init(address=ray_address,
79 + ignore_reinit_error=True,
80 + num_gpus=parallel_config.world_size)
81 + else:
82 + ray.init(address=ray_address, ignore_reinit_error=True)
83
84 if not parallel_config.worker_use_ray:
85 # Initialize cluster locally.
vllm/model_executor/layers/attention.py
+3
@@ -10,6 +10,7 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
10 from vllm._C import ops
11 from vllm._C import cache_ops
12 from vllm.model_executor.input_metadata import InputMetadata
13 +from vllm.utils import is_hip
14
15 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
16 # Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -160,6 +161,8 @@ class PagedAttention(nn.Module):
161 attn_bias=input_metadata.attn_bias,
162 p=0.0,
163 scale=self.scale,
164 + op=xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if
165 + (is_hip()) else None,
166 )
167 output = out.view_as(query)
168 else:
vllm/model_executor/layers/quantization/squeezellm.py
+9 -3
@@ -7,6 +7,7 @@ from vllm._C import ops
7 from vllm.model_executor.layers.linear import (LinearMethodBase,
8 set_weight_attrs)
9 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
10 +from vllm.utils import is_hip
11
12
13 class SqueezeLLMConfig(QuantizationConfig):
@@ -114,9 +115,14 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
115 lookup_table = weights["lookup_table"]
116 out_shape = x.shape[:-1] + (qweight.shape[-1], )
117 reshaped_x = x.reshape(-1, x.shape[-1])
117 - # NOTE: The output tensor should be zero-initialized.
118 - out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
119 - ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
118 + if is_hip():
119 + out_f = torch.zeros(out_shape, device="cuda", dtype=torch.float)
120 + ops.squeezellm_gemm(reshaped_x, qweight, out_f, lookup_table)
121 + out = out_f.to(dtype=torch.float16)
122 + else:
123 + # NOTE: The output tensor should be zero-initialized.
124 + out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
125 + ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
126
127 if bias is not None:
128 out = out + bias
vllm/model_executor/model_loader.py
+24
@@ -10,6 +10,10 @@ from vllm.config import ModelConfig
10 from vllm.model_executor.models import *
11 from vllm.model_executor.weight_utils import (get_quant_config,
12 initialize_dummy_weights)
13 +from vllm.utils import is_hip
14 +from vllm.logger import init_logger
15 +
16 +logger = init_logger(__name__)
17
18 # TODO(woosuk): Lazy-load the model classes.
19 _MODEL_REGISTRY = {
@@ -39,6 +43,18 @@ _MODEL_REGISTRY = {
43 "YiForCausalLM": YiForCausalLM,
44 }
45
46 +# Models to be disabled in ROCm
47 +_ROCM_UNSUPPORTED_MODELS = []
48 +if is_hip():
49 + for rocm_model in _ROCM_UNSUPPORTED_MODELS:
50 + del _MODEL_REGISTRY[rocm_model]
51 +
52 +# Models partially supported in ROCm
53 +_ROCM_PARTIALLY_SUPPORTED_MODELS = {
54 + "MistralForCausalLM":
55 + "Sliding window attention is not supported in ROCm's flash attention",
56 +}
57 +
58
59 @contextlib.contextmanager
60 def _set_default_torch_dtype(dtype: torch.dtype):
@@ -53,7 +69,15 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
69 architectures = getattr(config, "architectures", [])
70 for arch in architectures:
71 if arch in _MODEL_REGISTRY:
72 + if is_hip() and arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
73 + logger.warning(
74 + f"{arch} is not fully supported in ROCm. Reason: "
75 + f"{_ROCM_PARTIALLY_SUPPORTED_MODELS[arch]}")
76 return _MODEL_REGISTRY[arch]
77 + elif arch in _ROCM_UNSUPPORTED_MODELS:
78 + raise ValueError(
79 + f"Model architecture {arch} is not supported by ROCm for now. \n"
80 + f"Supported architectures {list(_MODEL_REGISTRY.keys())}")
81 raise ValueError(
82 f"Model architectures {architectures} are not supported for now. "
83 f"Supported architectures: {list(_MODEL_REGISTRY.keys())}")
vllm/utils.py
+5 -1
@@ -27,10 +27,14 @@ class Counter:
27 self.counter = 0
28
29
30 +def is_hip() -> bool:
31 + return torch.version.hip is not None
32 +
33 +
34 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
35 """Returns the maximum shared memory per thread block in bytes."""
36 # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
33 - cudaDevAttrMaxSharedMemoryPerBlockOptin = 97
37 + cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 if not is_hip() else 74
38 max_shared_mem = cuda_utils.get_device_attribute(
39 cudaDevAttrMaxSharedMemoryPerBlockOptin, gpu)
40 return int(max_shared_mem)