@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e070829ae8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Support bfloat16 data type (#54)

Woosuk Kwon committed May 3, 2023 at 14:09 UTC e070829ae81dc59cb660cf462a55fb703cd997e7
12 files changed +455 -53
cacheflow/master/server.py
+2 -2
@@ -213,8 +213,8 @@ def add_server_arguments(parser: argparse.ArgumentParser):
213 parser.add_argument('--use-np-cache', action='store_true',
214 help='save a numpy copy of model weights for faster loading')
215 parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216 - # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
217 - parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
216 + # NOTE(woosuk): FlashAttention does not support float32.
217 + parser.add_argument('--dtype', type=str, default='half', choices=['half', 'bfloat16'], help='data type')
218 # Parallel arguments
219 parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
220 parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
cacheflow/models/utils.py
+1
@@ -17,6 +17,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
17 'float': torch.float,
18 'float16': torch.float16,
19 'float32': torch.float32,
20 + 'bfloat16': torch.bfloat16,
21 }
22
23
csrc/activation_kernels.cu
+3 -1
@@ -34,7 +34,9 @@ void silu_and_mul(
34 dim3 grid(num_tokens);
35 dim3 block(std::min(d, 1024));
36 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
37 - AT_DISPATCH_FLOATING_TYPES_AND_HALF(
37 + AT_DISPATCH_FLOATING_TYPES_AND2(
38 + at::ScalarType::Half,
39 + at::ScalarType::BFloat16,
40 input.scalar_type(),
41 "silu_and_mul_kernel",
42 [&] {
csrc/attention/attention_dtypes.h renamed
+4
@@ -3,3 +3,7 @@
3 #include "attention_generic.cuh"
4 #include "dtype_float16.cuh"
5 #include "dtype_float32.cuh"
6 +
7 +#ifdef ENABLE_BF16
8 +#include "dtype_bfloat16.cuh"
9 +#endif // ENABLE_BF16
csrc/attention/attention_kernels.cu
+6 -2
@@ -1,7 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 -#include "attention_dtypes.cuh"
4 +#include "attention_dtypes.h"
5 #include "attention_utils.cuh"
6
7 #include <algorithm>
@@ -438,9 +438,13 @@ void single_query_cached_kv_attention(
438 torch::Tensor& context_lens, // [num_seqs]
439 int block_size,
440 int max_context_len) {
441 - // TODO(woosuk): Support FP32 and BF16.
441 + // TODO(woosuk): Support FP32.
442 if (query.dtype() == at::ScalarType::Half) {
443 CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
444 +#ifdef ENABLE_BF16
445 + } else if (query.dtype() == at::ScalarType::BFloat16) {
446 + CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
447 +#endif
448 } else {
449 TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
450 }
csrc/attention/attention_utils.cuh
+1 -1
@@ -1,6 +1,6 @@
1 #pragma once
2
3 -#include "attention_dtypes.cuh"
3 +#include "attention_dtypes.h"
4
5 #include <float.h>
6 #include <type_traits>
csrc/attention/dtype_bfloat16.cuh new
+361
@@ -0,0 +1,361 @@
1 +#pragma once
2 +
3 +#include "attention_generic.cuh"
4 +#include "dtype_float32.cuh"
5 +
6 +#include <cuda_bf16.h>
7 +#include <cuda_fp16.h>
8 +#include <stdint.h>
9 +
10 +namespace cacheflow {
11 +
12 +// Define custom BF16 vector data types.
13 +struct bf16_4_t {
14 + __nv_bfloat162 x;
15 + __nv_bfloat162 y;
16 +};
17 +
18 +struct bf16_8_t {
19 + __nv_bfloat162 x;
20 + __nv_bfloat162 y;
21 + __nv_bfloat162 z;
22 + __nv_bfloat162 w;
23 +};
24 +
25 +// BF16 vector types for Q, K, V.
26 +template<>
27 +struct Vec<__nv_bfloat16, 1> {
28 + using Type = __nv_bfloat16;
29 +};
30 +template<>
31 +struct Vec<__nv_bfloat16, 2> {
32 + using Type = __nv_bfloat162;
33 +};
34 +template<>
35 +struct Vec<__nv_bfloat16, 4> {
36 + using Type = bf16_4_t;
37 +};
38 +template<>
39 +struct Vec<__nv_bfloat16, 8> {
40 + using Type = bf16_8_t;
41 +};
42 +
43 +// FP32 accumulator vector types corresponding to Vec.
44 +template<>
45 +struct FloatVec<__nv_bfloat16> {
46 + using Type = float;
47 +};
48 +template<>
49 +struct FloatVec<__nv_bfloat162> {
50 + using Type = float2;
51 +};
52 +template<>
53 +struct FloatVec<bf16_4_t> {
54 + using Type = Float4_;
55 +};
56 +template<>
57 +struct FloatVec<bf16_8_t> {
58 + using Type = Float8_;
59 +};
60 +
61 +// Utility functions for type conversions.
62 +inline __device__ float2 bf1622float2(const __nv_bfloat162 val) {
63 + return __bfloat1622float2(val);
64 +}
65 +
66 +inline __device__ __nv_bfloat162 bf162bf162(const __nv_bfloat16 val) {
67 + return __bfloat162bfloat162(val);
68 +}
69 +
70 +// Vector addition.
71 +inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
72 + return a + b;
73 +}
74 +
75 +inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b) {
76 + return __hadd2(a, b);
77 +}
78 +
79 +inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b) {
80 + bf16_4_t c;
81 + c.x = add(a.x, b.x);
82 + c.y = add(a.y, b.y);
83 + return c;
84 +}
85 +
86 +inline __device__ bf16_8_t add(bf16_8_t a, bf16_8_t b) {
87 + bf16_8_t c;
88 + c.x = add(a.x, b.x);
89 + c.y = add(a.y, b.y);
90 + c.z = add(a.z, b.z);
91 + c.w = add(a.w, b.w);
92 + return c;
93 +}
94 +
95 +inline __device__ float2 add(__nv_bfloat162 a, float2 fb) {
96 + float2 fa = bf1622float2(a);
97 + return add(fa, fb);
98 +}
99 +
100 +inline __device__ Float4_ add(bf16_4_t a, Float4_ fb) {
101 + Float4_ fc;
102 + fc.x = add(a.x, fb.x);
103 + fc.y = add(a.y, fb.y);
104 + return fc;
105 +}
106 +
107 +inline __device__ Float8_ add(bf16_8_t a, Float8_ fb) {
108 + Float8_ fc;
109 + fc.x = add(a.x, fb.x);
110 + fc.y = add(a.y, fb.y);
111 + fc.z = add(a.z, fb.z);
112 + fc.w = add(a.w, fb.w);
113 + return fc;
114 +}
115 +
116 +// Vector multiplication.
117 +template<>
118 +inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
119 + return __hmul(a, b);
120 +}
121 +
122 +template<>
123 +inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
124 + return __hmul2(a, b);
125 +}
126 +
127 +template<>
128 +inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
129 + return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
130 +}
131 +
132 +template<>
133 +inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b) {
134 + bf16_4_t c;
135 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
136 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
137 + return c;
138 +}
139 +
140 +template<>
141 +inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b) {
142 + __nv_bfloat162 s = bf162bf162(a);
143 + bf16_4_t c;
144 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
145 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
146 + return c;
147 +}
148 +
149 +template<>
150 +inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b) {
151 + bf16_8_t c;
152 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
153 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
154 + c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
155 + c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
156 + return c;
157 +}
158 +
159 +template<>
160 +inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b) {
161 + __nv_bfloat162 s = bf162bf162(a);
162 + bf16_8_t c;
163 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
164 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
165 + c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.z);
166 + c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.w);
167 + return c;
168 +}
169 +
170 +template<>
171 +inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b) {
172 + float fa = __bfloat162float(a);
173 + float fb = __bfloat162float(b);
174 + return fa * fb;
175 +}
176 +
177 +template<>
178 +inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
179 + float2 fa = bf1622float2(a);
180 + float2 fb = bf1622float2(b);
181 + return mul<float2, float2, float2>(fa, fb);
182 +}
183 +
184 +template<>
185 +inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
186 + return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
187 +}
188 +
189 +template<>
190 +inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b) {
191 + Float4_ fc;
192 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
193 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
194 + return fc;
195 +}
196 +
197 +template<>
198 +inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b) {
199 + __nv_bfloat162 s = bf162bf162(a);
200 + Float4_ fc;
201 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
202 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
203 + return fc;
204 +}
205 +
206 +template<>
207 +inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b) {
208 + Float8_ fc;
209 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
210 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
211 + fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
212 + fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
213 + return fc;
214 +}
215 +
216 +template<>
217 +inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
218 + __nv_bfloat162 s = bf162bf162(a);
219 + Float8_ fc;
220 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
221 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
222 + fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.z);
223 + fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.w);
224 + return fc;
225 +}
226 +
227 +// Vector fused multiply-add.
228 +inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c) {
229 + return __hfma2(a, b, c);
230 +}
231 +
232 +inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c) {
233 + return __hfma2(bf162bf162(a), b, c);
234 +}
235 +
236 +inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c) {
237 + bf16_4_t d;
238 + d.x = fma(a.x, b.x, c.x);
239 + d.y = fma(a.y, b.y, c.y);
240 + return d;
241 +}
242 +
243 +inline __device__ bf16_4_t fma(__nv_bfloat16 a, bf16_4_t b, bf16_4_t c) {
244 + __nv_bfloat162 s = bf162bf162(a);
245 + bf16_4_t d;
246 + d.x = fma(s, b.x, c.x);
247 + d.y = fma(s, b.y, c.y);
248 + return d;
249 +}
250 +
251 +inline __device__ bf16_8_t fma(bf16_8_t a, bf16_8_t b, bf16_8_t c) {
252 + bf16_8_t d;
253 + d.x = fma(a.x, b.x, c.x);
254 + d.y = fma(a.y, b.y, c.y);
255 + d.z = fma(a.z, b.z, c.z);
256 + d.w = fma(a.w, b.w, c.w);
257 + return d;
258 +}
259 +
260 +inline __device__ bf16_8_t fma(__nv_bfloat16 a, bf16_8_t b, bf16_8_t c) {
261 + __nv_bfloat162 s = bf162bf162(a);
262 + bf16_8_t d;
263 + d.x = fma(s, b.x, c.x);
264 + d.y = fma(s, b.y, c.y);
265 + d.z = fma(s, b.z, c.z);
266 + d.w = fma(s, b.w, c.w);
267 + return d;
268 +}
269 +
270 +inline __device__ float fma(__nv_bfloat16 a, __nv_bfloat16 b, float fc) {
271 + return __bfloat162float(a) * __bfloat162float(b) + fc;
272 +}
273 +
274 +inline __device__ float2 fma(__nv_bfloat162 a, __nv_bfloat162 b, float2 fc) {
275 + float2 fa = bf1622float2(a);
276 + float2 fb = bf1622float2(b);
277 + return fma(fa, fb, fc);
278 +}
279 +
280 +inline __device__ float2 fma(__nv_bfloat16 a, __nv_bfloat162 b, float2 fc) {
281 + return fma(bf162bf162(a), b, fc);
282 +}
283 +
284 +inline __device__ Float4_ fma(bf16_4_t a, bf16_4_t b, Float4_ fc) {
285 + Float4_ fd;
286 + fd.x = fma(a.x, b.x, fc.x);
287 + fd.y = fma(a.y, b.y, fc.y);
288 + return fd;
289 +}
290 +
291 +inline __device__ Float4_ fma(__nv_bfloat16 a, bf16_4_t b, Float4_ fc) {
292 + __nv_bfloat162 s = bf162bf162(a);
293 + Float4_ fd;
294 + fd.x = fma(s, b.x, fc.x);
295 + fd.y = fma(s, b.y, fc.y);
296 + return fd;
297 +}
298 +
299 +inline __device__ Float8_ fma(bf16_8_t a, bf16_8_t b, Float8_ fc) {
300 + Float8_ fd;
301 + fd.x = fma(a.x, b.x, fc.x);
302 + fd.y = fma(a.y, b.y, fc.y);
303 + fd.z = fma(a.z, b.z, fc.z);
304 + fd.w = fma(a.w, b.w, fc.w);
305 + return fd;
306 +}
307 +
308 +inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc) {
309 + __nv_bfloat162 s = bf162bf162(a);
310 + Float8_ fd;
311 + fd.x = fma(s, b.x, fc.x);
312 + fd.y = fma(s, b.y, fc.y);
313 + fd.z = fma(s, b.z, fc.z);
314 + fd.w = fma(s, b.w, fc.w);
315 + return fd;
316 +}
317 +
318 +// Vector sum.
319 +template<>
320 +inline __device__ float sum(__nv_bfloat16 v) {
321 + return __bfloat162float(v);
322 +}
323 +
324 +template<>
325 +inline __device__ float sum(__nv_bfloat162 v) {
326 + float2 vf = bf1622float2(v);
327 + return vf.x + vf.y;
328 +}
329 +
330 +template<>
331 +inline __device__ float sum(bf16_4_t v) {
332 + return sum(v.x) + sum(v.y);
333 +}
334 +
335 +template<>
336 +inline __device__ float sum(bf16_8_t v) {
337 + return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
338 +}
339 +
340 +// From float32 to bfloat16.
341 +inline __device__ void from_float(__nv_bfloat16& dst, float src) {
342 + dst = __float2bfloat16(src);
343 +}
344 +
345 +inline __device__ void from_float(__nv_bfloat162& dst, float2 src) {
346 + dst = __float22bfloat162_rn(src);
347 +}
348 +
349 +inline __device__ void from_float(bf16_4_t& dst, Float4_ src) {
350 + dst.x = __float22bfloat162_rn(src.x);
351 + dst.y = __float22bfloat162_rn(src.y);
352 +}
353 +
354 +inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
355 + dst.x = __float22bfloat162_rn(src.x);
356 + dst.y = __float22bfloat162_rn(src.y);
357 + dst.z = __float22bfloat162_rn(src.z);
358 + dst.w = __float22bfloat162_rn(src.w);
359 +}
360 +
361 +} // namespace cacheflow
csrc/attention/dtype_float32.cuh
+1 -1
@@ -6,7 +6,7 @@
6
7 namespace cacheflow {
8
9 -// Define FP32 vector data types.
9 +// Define custom FP32 vector data types.
10 struct Float4_ {
11 float2 x;
12 float2 y;
csrc/cache_kernels.cu
+55 -43
@@ -14,14 +14,16 @@ void swap_blocks(
14 torch::Device dst_device = dst.device();
15 cudaMemcpyKind memcpy_type;
16 if (src_device.is_cuda() && dst_device.is_cuda()) {
17 - assert(src_device.index() == dst_device.index());
17 + TORCH_CHECK(
18 + src_device.index() == dst_device.index(),
19 + "src and dst must be on the same GPU");
20 memcpy_type = cudaMemcpyDeviceToDevice;
21 } else if (src_device.is_cuda() && dst_device.is_cpu()) {
22 memcpy_type = cudaMemcpyDeviceToHost;
23 } else if (src_device.is_cpu() && dst_device.is_cuda()) {
24 memcpy_type = cudaMemcpyHostToDevice;
25 } else {
24 - assert(false);
26 + TORCH_CHECK(false, "Invalid device combination");
27 }
28
29 void *src_ptr = src.data_ptr();
@@ -29,6 +31,7 @@ void swap_blocks(
31
32 const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
33 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
34 + // NOTE(woosuk): This can be slow if the number of blocks is large.
35 for (const auto& pair : block_mapping) {
36 int64_t src_block_number = pair.first;
37 int64_t dst_block_number = pair.second;
@@ -122,7 +125,9 @@ void copy_blocks(
125 dim3 grid(num_layers, num_pairs);
126 dim3 block(std::min(1024, numel_per_block));
127 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
125 - AT_DISPATCH_FLOATING_TYPES_AND_HALF(
128 + AT_DISPATCH_FLOATING_TYPES_AND2(
129 + at::ScalarType::Half,
130 + at::ScalarType::BFloat16,
131 key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
132 cacheflow::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
133 key_cache_ptrs_tensor.data_ptr<int64_t>(),
@@ -176,6 +181,50 @@ __global__ void reshape_and_cache_kernel(
181 }
182 }
183
184 +} // namespace cacheflow
185 +
186 +void reshape_and_cache(
187 + torch::Tensor& key, // [num_tokens, num_heads, head_size]
188 + torch::Tensor& value, // [num_tokens, num_heads, head_size]
189 + torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
190 + torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
191 + torch::Tensor& slot_mapping) // [num_tokens]
192 +{
193 + int num_tokens = key.size(0);
194 + int num_heads = key.size(1);
195 + int head_size = key.size(2);
196 + int block_size = key_cache.size(3);
197 + int x = key_cache.size(4);
198 +
199 + int key_stride = key.stride(0);
200 + int value_stride = value.stride(0);
201 +
202 + dim3 grid(num_tokens);
203 + dim3 block(std::min(num_heads * head_size, 512));
204 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
205 + AT_DISPATCH_FLOATING_TYPES_AND2(
206 + at::ScalarType::Half,
207 + at::ScalarType::BFloat16,
208 + key.scalar_type(),
209 + "reshape_and_cache_kernel",
210 + [&] {
211 + cacheflow::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
212 + key.data_ptr<scalar_t>(),
213 + value.data_ptr<scalar_t>(),
214 + key_cache.data_ptr<scalar_t>(),
215 + value_cache.data_ptr<scalar_t>(),
216 + slot_mapping.data_ptr<int>(),
217 + key_stride,
218 + value_stride,
219 + num_heads,
220 + head_size,
221 + block_size,
222 + x);
223 + });
224 +}
225 +
226 +namespace cacheflow {
227 +
228 // Grid: (num_blocks, block_size).
229 template<typename scalar_t>
230 __global__ void gather_cached_kv_kernel(
@@ -296,45 +345,6 @@ __global__ void gather_cached_kv_kernel_optimized(
345
346 } // namespace cacheflow
347
299 -void reshape_and_cache(
300 - torch::Tensor& key, // [num_tokens, num_heads, head_size]
301 - torch::Tensor& value, // [num_tokens, num_heads, head_size]
302 - torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
303 - torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
304 - torch::Tensor& slot_mapping) // [num_tokens]
305 -{
306 - int num_tokens = key.size(0);
307 - int num_heads = key.size(1);
308 - int head_size = key.size(2);
309 - int block_size = key_cache.size(3);
310 - int x = key_cache.size(4);
311 -
312 - int key_stride = key.stride(0);
313 - int value_stride = value.stride(0);
314 -
315 - dim3 grid(num_tokens);
316 - dim3 block(std::min(num_heads * head_size, 512));
317 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
318 - AT_DISPATCH_FLOATING_TYPES_AND_HALF(
319 - key.scalar_type(),
320 - "reshape_and_cache_kernel",
321 - [&] {
322 - cacheflow::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
323 - key.data_ptr<scalar_t>(),
324 - value.data_ptr<scalar_t>(),
325 - key_cache.data_ptr<scalar_t>(),
326 - value_cache.data_ptr<scalar_t>(),
327 - slot_mapping.data_ptr<int>(),
328 - key_stride,
329 - value_stride,
330 - num_heads,
331 - head_size,
332 - block_size,
333 - x);
334 - });
335 -}
336 -
337 -
348 void gather_cached_kv(
349 torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
350 torch::Tensor& value, // [out] [num_tokens, num_heads, head_size]
@@ -354,7 +364,9 @@ void gather_cached_kv(
364 dim3 grid(num_tokens);
365 dim3 block(std::min(num_heads * head_size, 512));
366 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
357 - AT_DISPATCH_FLOATING_TYPES_AND_HALF(
367 + AT_DISPATCH_FLOATING_TYPES_AND2(
368 + at::ScalarType::Half,
369 + at::ScalarType::BFloat16,
370 key.scalar_type(),
371 "gather_cached_kv_kernel_optimized",
372 [&] {
csrc/layernorm_kernels.cu
+3 -1
@@ -46,7 +46,9 @@ void rms_norm(
46 dim3 grid(num_tokens);
47 dim3 block(std::min(hidden_size, 1024));
48 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
49 - AT_DISPATCH_FLOATING_TYPES_AND_HALF(
49 + AT_DISPATCH_FLOATING_TYPES_AND2(
50 + at::ScalarType::Half,
51 + at::ScalarType::BFloat16,
52 input.scalar_type(),
53 "rms_norm_kernel",
54 [&] {
csrc/pos_encoding_kernels.cu
+3 -1
@@ -64,7 +64,9 @@ void rotary_embedding_neox(
64 dim3 grid(num_tokens);
65 dim3 block(std::min(num_heads * rot_dim / 2, 512));
66 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
67 - AT_DISPATCH_FLOATING_TYPES_AND_HALF(
67 + AT_DISPATCH_FLOATING_TYPES_AND2(
68 + at::ScalarType::Half,
69 + at::ScalarType::BFloat16,
70 query.scalar_type(),
71 "rotary_embedding_neox",
72 [&] {
setup.py
+15 -1
@@ -1,9 +1,22 @@
1 import setuptools
2 +import torch
3 from torch.utils import cpp_extension
4
5 CXX_FLAGS = ['-g']
6 NVCC_FLAGS = ['-O2']
7
8 +if not torch.cuda.is_available():
9 + raise RuntimeError(
10 + f'Cannot find CUDA at CUDA_HOME: {cpp_extension.CUDA_HOME}. '
11 + 'CUDA must be available in order to build the package.')
12 +
13 +# FIXME(woosuk): Consider the case where the machine has multiple GPUs with
14 +# different compute capabilities.
15 +compute_capability = torch.cuda.get_device_capability()
16 +major, minor = compute_capability
17 +# Enable bfloat16 support if the compute capability is >= 8.0.
18 +if major >= 8:
19 + NVCC_FLAGS.append('-DENABLE_BF16')
20
21 ext_modules = []
22
@@ -23,7 +36,7 @@ attention_extension = cpp_extension.CUDAExtension(
36 )
37 ext_modules.append(attention_extension)
38
26 -# Positional encodings.
39 +# Positional encoding kernels.
40 positional_encoding_extension = cpp_extension.CUDAExtension(
41 name='cacheflow.pos_encoding_ops',
42 sources=['csrc/pos_encoding.cpp', 'csrc/pos_encoding_kernels.cu'],
@@ -39,6 +52,7 @@ layernorm_extension = cpp_extension.CUDAExtension(
52 )
53 ext_modules.append(layernorm_extension)
54
55 +# Activation kernels.
56 activation_extension = cpp_extension.CUDAExtension(
57 name='cacheflow.activation_ops',
58 sources=['csrc/activation.cpp', 'csrc/activation_kernels.cu'],