@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d721168449
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Improve setup script & Add a guard for bfloat16 kernels (#130)

Woosuk Kwon committed May 27, 2023 at 00:59 UTC d721168449281a5f5a299bba8d94523ad4e833cb
4 files changed +90 -16
csrc/attention/attention_dtypes.h
-3
@@ -3,7 +3,4 @@
3 #include "attention_generic.cuh"
4 #include "dtype_float16.cuh"
5 #include "dtype_float32.cuh"
6 -
7 -#ifdef ENABLE_BF16
6 #include "dtype_bfloat16.cuh"
9 -#endif // ENABLE_BF16
csrc/attention/attention_kernels.cu
-2
@@ -458,10 +458,8 @@ void single_query_cached_kv_attention(
458 // TODO(woosuk): Support FP32.
459 if (query.dtype() == at::ScalarType::Half) {
460 CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
461 -#ifdef ENABLE_BF16
461 } else if (query.dtype() == at::ScalarType::BFloat16) {
462 CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
464 -#endif
463 } else {
464 TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
465 }
csrc/attention/dtype_bfloat16.cuh
+44
@@ -78,20 +78,36 @@ struct FloatVec<bf16_8_t> {
78
79 // Utility functions for type conversions.
80 inline __device__ float2 bf1622float2(const __nv_bfloat162 val) {
81 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
82 + assert(false);
83 +#else
84 return __bfloat1622float2(val);
85 +#endif
86 }
87
88 inline __device__ __nv_bfloat162 bf162bf162(const __nv_bfloat16 val) {
89 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
90 + assert(false);
91 +#else
92 return __bfloat162bfloat162(val);
93 +#endif
94 }
95
96 // Vector addition.
97 inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
98 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
99 + assert(false);
100 +#else
101 return a + b;
102 +#endif
103 }
104
105 inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b) {
106 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
107 + assert(false);
108 +#else
109 return __hadd2(a, b);
110 +#endif
111 }
112
113 inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b) {
@@ -134,12 +150,20 @@ inline __device__ Float8_ add(bf16_8_t a, Float8_ fb) {
150 // Vector multiplication.
151 template<>
152 inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
153 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
154 + assert(false);
155 +#else
156 return __hmul(a, b);
157 +#endif
158 }
159
160 template<>
161 inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
162 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
163 + assert(false);
164 +#else
165 return __hmul2(a, b);
166 +#endif
167 }
168
169 template<>
@@ -244,11 +268,19 @@ inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
268
269 // Vector fused multiply-add.
270 inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c) {
271 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
272 + assert(false);
273 +#else
274 return __hfma2(a, b, c);
275 +#endif
276 }
277
278 inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c) {
279 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
280 + assert(false);
281 +#else
282 return __hfma2(bf162bf162(a), b, c);
283 +#endif
284 }
285
286 inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c) {
@@ -361,19 +393,31 @@ inline __device__ void from_float(__nv_bfloat16& dst, float src) {
393 }
394
395 inline __device__ void from_float(__nv_bfloat162& dst, float2 src) {
396 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
397 + assert(false);
398 +#else
399 dst = __float22bfloat162_rn(src);
400 +#endif
401 }
402
403 inline __device__ void from_float(bf16_4_t& dst, Float4_ src) {
404 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
405 + assert(false);
406 +#else
407 dst.x = __float22bfloat162_rn(src.x);
408 dst.y = __float22bfloat162_rn(src.y);
409 +#endif
410 }
411
412 inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
413 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
414 + assert(false);
415 +#else
416 dst.x = __float22bfloat162_rn(src.x);
417 dst.y = __float22bfloat162_rn(src.y);
418 dst.z = __float22bfloat162_rn(src.z);
419 dst.w = __float22bfloat162_rn(src.w);
420 +#endif
421 }
422
423 } // namespace cacheflow
setup.py
+46 -11
@@ -1,28 +1,63 @@
1 -from typing import List
1 +import subprocess
2 +from typing import List, Set
3
4 +from packaging.version import parse, Version
5 import setuptools
6 import torch
7 from torch.utils.cpp_extension import BuildExtension, CUDAExtension
8 from torch.utils.cpp_extension import CUDA_HOME
9
8 -
9 -# Build custom operators.
10 -CXX_FLAGS = ["-g"]
10 +# Compiler flags.
11 +CXX_FLAGS = ["-g", "-O2"]
12 # TODO(woosuk): Should we use -O3?
13 NVCC_FLAGS = ["-O2"]
14
15 +
16 if not torch.cuda.is_available():
17 raise RuntimeError(
18 f"Cannot find CUDA at CUDA_HOME: {CUDA_HOME}. "
19 "CUDA must be available in order to build the package.")
20
19 -# FIXME(woosuk): Consider the case where the machine has multiple GPUs with
20 -# different compute capabilities.
21 -compute_capability = torch.cuda.get_device_capability()
22 -major, minor = compute_capability
23 -# Enable bfloat16 support if the compute capability is >= 8.0.
24 -if major >= 8:
25 - NVCC_FLAGS.append("-DENABLE_BF16")
21 +
22 +def get_nvcc_cuda_version(cuda_dir: str) -> Version:
23 + """Get the CUDA version from nvcc.
24 +
25 + Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
26 + """
27 + nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
28 + universal_newlines=True)
29 + output = nvcc_output.split()
30 + release_idx = output.index("release") + 1
31 + nvcc_cuda_version = parse(output[release_idx].split(",")[0])
32 + return nvcc_cuda_version
33 +
34 +
35 +# Collect the compute capabilities of all available GPUs.
36 +device_count = torch.cuda.device_count()
37 +compute_capabilities: Set[int] = set()
38 +for i in range(device_count):
39 + major, minor = torch.cuda.get_device_capability(i)
40 + if major < 7:
41 + raise RuntimeError(
42 + "GPUs with compute capability less than 7.0 are not supported.")
43 + compute_capabilities.add(major * 10 + minor)
44 +# If no GPU is available, add all supported compute capabilities.
45 +if not compute_capabilities:
46 + compute_capabilities = {70, 75, 80, 86, 90}
47 +# Add target compute capabilities to NVCC flags.
48 +for capability in compute_capabilities:
49 + NVCC_FLAGS += ["-gencode", f"arch=compute_{capability},code=sm_{capability}"]
50 +
51 +# Validate the NVCC CUDA version.
52 +nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
53 +if nvcc_cuda_version < Version("11.0"):
54 + raise RuntimeError("CUDA 11.0 or higher is required to build the package.")
55 +if 86 in compute_capabilities and nvcc_cuda_version < Version("11.1"):
56 + raise RuntimeError(
57 + "CUDA 11.1 or higher is required for GPUs with compute capability 8.6.")
58 +if 90 in compute_capabilities and nvcc_cuda_version < Version("11.8"):
59 + raise RuntimeError(
60 + "CUDA 11.8 or higher is required for GPUs with compute capability 9.0.")
61
62 ext_modules = []
63