A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix compile error when using rocm (#2648)
zhaoyang-star committed
Feb 2, 2024 at 01:35 UTC
923797fea4d80a4dac4409ece3c450b84d5ba001
3 files changed
+9
-1
csrc/attention/attention_kernels.cu
+2
@@ -25,7 +25,9 @@
25
26
#include "attention_dtypes.h"
27
#include "attention_utils.cuh"
28
+#ifdef ENABLE_FP8_E5M2
29
#include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
30
+#endif
31
32
#include <algorithm>
33
csrc/cache_kernels.cu
+7
@@ -4,13 +4,20 @@
4
5
#include "cuda_compat.h"
6
#include "dispatch_utils.h"
7
+#ifdef ENABLE_FP8_E5M2
8
#include "quantization/fp8_e5m2_kvcache/quant_utils.cuh"
9
+#endif
10
11
#include <algorithm>
12
#include <cassert>
13
#include <map>
14
#include <vector>
15
16
+#ifdef USE_ROCM
17
+ #include <hip/hip_bf16.h>
18
+ typedef __hip_bfloat16 __nv_bfloat16;
19
+#endif
20
+
21
void swap_blocks(
22
torch::Tensor& src,
23
torch::Tensor& dst,
csrc/quantization/fp8_e5m2_kvcache/quant_utils.cuh
-1
@@ -9,7 +9,6 @@
9
#include "../../attention/dtype_float16.cuh"
10
#include "../../attention/dtype_bfloat16.cuh"
11
12
-#pragma once
12
13
namespace vllm {
14
#ifdef ENABLE_FP8_E5M2