A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Improve setup script & Add a guard for bfloat16 kernels (#130)
Woosuk Kwon committed
May 27, 2023 at 00:59 UTC
d721168449281a5f5a299bba8d94523ad4e833cb
4 files changed
+90
-16
csrc/attention/attention_dtypes.h
-3
@@ -3,7 +3,4 @@
3
#include "attention_generic.cuh"
4
#include "dtype_float16.cuh"
5
#include "dtype_float32.cuh"
6
-
7
-#ifdef ENABLE_BF16
6
#include "dtype_bfloat16.cuh"
9
-#endif // ENABLE_BF16
csrc/attention/attention_kernels.cu
-2
@@ -458,10 +458,8 @@ void single_query_cached_kv_attention(
458
// TODO(woosuk): Support FP32.
459
if (query.dtype() == at::ScalarType::Half) {
460
CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
461
-#ifdef ENABLE_BF16
461
} else if (query.dtype() == at::ScalarType::BFloat16) {
462
CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
464
-#endif
463
} else {
464
TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
465
}
csrc/attention/dtype_bfloat16.cuh
+44
@@ -78,20 +78,36 @@ struct FloatVec<bf16_8_t> {
78
79
// Utility functions for type conversions.
80
inline __device__ float2 bf1622float2(const __nv_bfloat162 val) {
81
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
82
+ assert(false);
83
+#else
84
return __bfloat1622float2(val);
85
+#endif
86
}
87
88
inline __device__ __nv_bfloat162 bf162bf162(const __nv_bfloat16 val) {
89
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
90
+ assert(false);
91
+#else
92
return __bfloat162bfloat162(val);
93
+#endif
94
}
95
96
// Vector addition.
97
inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
98
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
99
+ assert(false);
100
+#else
101
return a + b;
102
+#endif
103
}
104
105
inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b) {
106
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
107
+ assert(false);
108
+#else
109
return __hadd2(a, b);
110
+#endif
111
}
112
113
inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b) {
@@ -134,12 +150,20 @@ inline __device__ Float8_ add(bf16_8_t a, Float8_ fb) {
150
// Vector multiplication.
151
template<>
152
inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
153
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
154
+ assert(false);
155
+#else
156
return __hmul(a, b);
157
+#endif
158
}
159
160
template<>
161
inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
162
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
163
+ assert(false);
164
+#else
165
return __hmul2(a, b);
166
+#endif
167
}
168
169
template<>
@@ -244,11 +268,19 @@ inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
268
269
// Vector fused multiply-add.
270
inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c) {
271
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
272
+ assert(false);
273
+#else
274
return __hfma2(a, b, c);
275
+#endif
276
}
277
278
inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c) {
279
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
280
+ assert(false);
281
+#else
282
return __hfma2(bf162bf162(a), b, c);
283
+#endif
284
}
285
286
inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c) {
@@ -361,19 +393,31 @@ inline __device__ void from_float(__nv_bfloat16& dst, float src) {
393
}
394
395
inline __device__ void from_float(__nv_bfloat162& dst, float2 src) {
396
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
397
+ assert(false);
398
+#else
399
dst = __float22bfloat162_rn(src);
400
+#endif
401
}
402
403
inline __device__ void from_float(bf16_4_t& dst, Float4_ src) {
404
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
405
+ assert(false);
406
+#else
407
dst.x = __float22bfloat162_rn(src.x);
408
dst.y = __float22bfloat162_rn(src.y);
409
+#endif
410
}
411
412
inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
413
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
414
+ assert(false);
415
+#else
416
dst.x = __float22bfloat162_rn(src.x);
417
dst.y = __float22bfloat162_rn(src.y);
418
dst.z = __float22bfloat162_rn(src.z);
419
dst.w = __float22bfloat162_rn(src.w);
420
+#endif
421
}
422
423
} // namespace cacheflow
setup.py
+46
-11
@@ -1,28 +1,63 @@
1
-from typing import List
1
+import subprocess
2
+from typing import List, Set
3
4
+from packaging.version import parse, Version
5
import setuptools
6
import torch
7
from torch.utils.cpp_extension import BuildExtension, CUDAExtension
8
from torch.utils.cpp_extension import CUDA_HOME
9
8
-
9
-# Build custom operators.
10
-CXX_FLAGS = ["-g"]
10
+# Compiler flags.
11
+CXX_FLAGS = ["-g", "-O2"]
12
# TODO(woosuk): Should we use -O3?
13
NVCC_FLAGS = ["-O2"]
14
15
+
16
if not torch.cuda.is_available():
17
raise RuntimeError(
18
f"Cannot find CUDA at CUDA_HOME: {CUDA_HOME}. "
19
"CUDA must be available in order to build the package.")
20
19
-# FIXME(woosuk): Consider the case where the machine has multiple GPUs with
20
-# different compute capabilities.
21
-compute_capability = torch.cuda.get_device_capability()
22
-major, minor = compute_capability
23
-# Enable bfloat16 support if the compute capability is >= 8.0.
24
-if major >= 8:
25
- NVCC_FLAGS.append("-DENABLE_BF16")
21
+
22
+def get_nvcc_cuda_version(cuda_dir: str) -> Version:
23
+ """Get the CUDA version from nvcc.
24
+
25
+ Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
26
+ """
27
+ nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
28
+ universal_newlines=True)
29
+ output = nvcc_output.split()
30
+ release_idx = output.index("release") + 1
31
+ nvcc_cuda_version = parse(output[release_idx].split(",")[0])
32
+ return nvcc_cuda_version
33
+
34
+
35
+# Collect the compute capabilities of all available GPUs.
36
+device_count = torch.cuda.device_count()
37
+compute_capabilities: Set[int] = set()
38
+for i in range(device_count):
39
+ major, minor = torch.cuda.get_device_capability(i)
40
+ if major < 7:
41
+ raise RuntimeError(
42
+ "GPUs with compute capability less than 7.0 are not supported.")
43
+ compute_capabilities.add(major * 10 + minor)
44
+# If no GPU is available, add all supported compute capabilities.
45
+if not compute_capabilities:
46
+ compute_capabilities = {70, 75, 80, 86, 90}
47
+# Add target compute capabilities to NVCC flags.
48
+for capability in compute_capabilities:
49
+ NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
50
+
51
+# Validate the NVCC CUDA version.
52
+nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
53
+if nvcc_cuda_version < Version("11.0"):
54
+ raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
55
+if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56
+ raise RuntimeError(
57
+ "CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
58
+if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
59
+ raise RuntimeError(
60
+ "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
61
62
ext_modules = []
63