@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 923797fea4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix compile error when using rocm (#2648)

zhaoyang-star committed Feb 2, 2024 at 01:35 UTC 923797fea4d80a4dac4409ece3c450b84d5ba001
3 files changed +9 -1
csrc/attention/attention_kernels.cu
+2
@@ -25,7 +25,9 @@
25
26 #include "attention_dtypes.h"
27 #include "attention_utils.cuh"
28 +#ifdef ENABLE_FP8_E5M2
29 #include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
30 +#endif
31
32 #include <algorithm>
33
csrc/cache_kernels.cu
+7
@@ -4,13 +4,20 @@
4
5 #include "cuda_compat.h"
6 #include "dispatch_utils.h"
7 +#ifdef ENABLE_FP8_E5M2
8 #include "quantization/fp8_e5m2_kvcache/quant_utils.cuh"
9 +#endif
10
11 #include <algorithm>
12 #include <cassert>
13 #include <map>
14 #include <vector>
15
16 +#ifdef USE_ROCM
17 + #include <hip/hip_bf16.h>
18 + typedef __hip_bfloat16 __nv_bfloat16;
19 +#endif
20 +
21 void swap_blocks(
22 torch::Tensor& src,
23 torch::Tensor& dst,
csrc/quantization/fp8_e5m2_kvcache/quant_utils.cuh
-1
@@ -9,7 +9,6 @@
9 #include "../../attention/dtype_float16.cuh"
10 #include "../../attention/dtype_bfloat16.cuh"
11
12 -#pragma once
12
13 namespace vllm {
14 #ifdef ENABLE_FP8_E5M2