A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Support bfloat16 data type (#54)
Woosuk Kwon committed
May 3, 2023 at 14:09 UTC
e070829ae81dc59cb660cf462a55fb703cd997e7
12 files changed
+455
-53
cacheflow/master/server.py
+2
-2
@@ -213,8 +213,8 @@ def add_server_arguments(parser: argparse.ArgumentParser):
213
parser.add_argument('--use-np-cache', action='store_true',
214
help='save a numpy copy of model weights for faster loading')
215
parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216
- # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
217
- parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
216
+ # NOTE(woosuk): FlashAttention does not support float32.
217
+ parser.add_argument('--dtype', type=str, default='half', choices=['half', 'bfloat16'], help='data type')
218
# Parallel arguments
219
parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
220
parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
cacheflow/models/utils.py
+1
@@ -17,6 +17,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
17
'float': torch.float,
18
'float16': torch.float16,
19
'float32': torch.float32,
20
+ 'bfloat16': torch.bfloat16,
21
}
22
23
csrc/activation_kernels.cu
+3
-1
@@ -34,7 +34,9 @@ void silu_and_mul(
34
dim3 grid(num_tokens);
35
dim3 block(std::min(d, 1024));
36
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
37
- AT_DISPATCH_FLOATING_TYPES_AND_HALF(
37
+ AT_DISPATCH_FLOATING_TYPES_AND2(
38
+ at::ScalarType::Half,
39
+ at::ScalarType::BFloat16,
40
input.scalar_type(),
41
"silu_and_mul_kernel",
42
[&] {
csrc/attention/attention_dtypes.h
renamed
+4
@@ -3,3 +3,7 @@
3
#include "attention_generic.cuh"
4
#include "dtype_float16.cuh"
5
#include "dtype_float32.cuh"
6
+
7
+#ifdef ENABLE_BF16
8
+#include "dtype_bfloat16.cuh"
9
+#endif // ENABLE_BF16
csrc/attention/attention_kernels.cu
+6
-2
@@ -1,7 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
-#include "attention_dtypes.cuh"
4
+#include "attention_dtypes.h"
5
#include "attention_utils.cuh"
6
7
#include <algorithm>
@@ -438,9 +438,13 @@ void single_query_cached_kv_attention(
438
torch::Tensor& context_lens, // [num_seqs]
439
int block_size,
440
int max_context_len) {
441
- // TODO(woosuk): Support FP32 and BF16.
441
+ // TODO(woosuk): Support FP32.
442
if (query.dtype() == at::ScalarType::Half) {
443
CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
444
+#ifdef ENABLE_BF16
445
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
446
+ CALL_KERNEL_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
447
+#endif
448
} else {
449
TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
450
}
csrc/attention/attention_utils.cuh
+1
-1
@@ -1,6 +1,6 @@
1
#pragma once
2
3
-#include "attention_dtypes.cuh"
3
+#include "attention_dtypes.h"
4
5
#include <float.h>
6
#include <type_traits>
csrc/attention/dtype_bfloat16.cuh
new
+361
@@ -0,0 +1,361 @@
1
+#pragma once
2
+
3
+#include "attention_generic.cuh"
4
+#include "dtype_float32.cuh"
5
+
6
+#include <cuda_bf16.h>
7
+#include <cuda_fp16.h>
8
+#include <stdint.h>
9
+
10
+namespace cacheflow {
11
+
12
+// Define custom BF16 vector data types.
13
+struct bf16_4_t {
14
+ __nv_bfloat162 x;
15
+ __nv_bfloat162 y;
16
+};
17
+
18
+struct bf16_8_t {
19
+ __nv_bfloat162 x;
20
+ __nv_bfloat162 y;
21
+ __nv_bfloat162 z;
22
+ __nv_bfloat162 w;
23
+};
24
+
25
+// BF16 vector types for Q, K, V.
26
+template<>
27
+struct Vec<__nv_bfloat16, 1> {
28
+ using Type = __nv_bfloat16;
29
+};
30
+template<>
31
+struct Vec<__nv_bfloat16, 2> {
32
+ using Type = __nv_bfloat162;
33
+};
34
+template<>
35
+struct Vec<__nv_bfloat16, 4> {
36
+ using Type = bf16_4_t;
37
+};
38
+template<>
39
+struct Vec<__nv_bfloat16, 8> {
40
+ using Type = bf16_8_t;
41
+};
42
+
43
+// FP32 accumulator vector types corresponding to Vec.
44
+template<>
45
+struct FloatVec<__nv_bfloat16> {
46
+ using Type = float;
47
+};
48
+template<>
49
+struct FloatVec<__nv_bfloat162> {
50
+ using Type = float2;
51
+};
52
+template<>
53
+struct FloatVec<bf16_4_t> {
54
+ using Type = Float4_;
55
+};
56
+template<>
57
+struct FloatVec<bf16_8_t> {
58
+ using Type = Float8_;
59
+};
60
+
61
+// Utility functions for type conversions.
62
+inline __device__ float2 bf1622float2(const __nv_bfloat162 val) {
63
+ return __bfloat1622float2(val);
64
+}
65
+
66
+inline __device__ __nv_bfloat162 bf162bf162(const __nv_bfloat16 val) {
67
+ return __bfloat162bfloat162(val);
68
+}
69
+
70
+// Vector addition.
71
+inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b) {
72
+ return a + b;
73
+}
74
+
75
+inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b) {
76
+ return __hadd2(a, b);
77
+}
78
+
79
+inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b) {
80
+ bf16_4_t c;
81
+ c.x = add(a.x, b.x);
82
+ c.y = add(a.y, b.y);
83
+ return c;
84
+}
85
+
86
+inline __device__ bf16_8_t add(bf16_8_t a, bf16_8_t b) {
87
+ bf16_8_t c;
88
+ c.x = add(a.x, b.x);
89
+ c.y = add(a.y, b.y);
90
+ c.z = add(a.z, b.z);
91
+ c.w = add(a.w, b.w);
92
+ return c;
93
+}
94
+
95
+inline __device__ float2 add(__nv_bfloat162 a, float2 fb) {
96
+ float2 fa = bf1622float2(a);
97
+ return add(fa, fb);
98
+}
99
+
100
+inline __device__ Float4_ add(bf16_4_t a, Float4_ fb) {
101
+ Float4_ fc;
102
+ fc.x = add(a.x, fb.x);
103
+ fc.y = add(a.y, fb.y);
104
+ return fc;
105
+}
106
+
107
+inline __device__ Float8_ add(bf16_8_t a, Float8_ fb) {
108
+ Float8_ fc;
109
+ fc.x = add(a.x, fb.x);
110
+ fc.y = add(a.y, fb.y);
111
+ fc.z = add(a.z, fb.z);
112
+ fc.w = add(a.w, fb.w);
113
+ return fc;
114
+}
115
+
116
+// Vector multiplication.
117
+template<>
118
+inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b) {
119
+ return __hmul(a, b);
120
+}
121
+
122
+template<>
123
+inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
124
+ return __hmul2(a, b);
125
+}
126
+
127
+template<>
128
+inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
129
+ return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
130
+}
131
+
132
+template<>
133
+inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b) {
134
+ bf16_4_t c;
135
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
136
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
137
+ return c;
138
+}
139
+
140
+template<>
141
+inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b) {
142
+ __nv_bfloat162 s = bf162bf162(a);
143
+ bf16_4_t c;
144
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
145
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
146
+ return c;
147
+}
148
+
149
+template<>
150
+inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b) {
151
+ bf16_8_t c;
152
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
153
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
154
+ c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
155
+ c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
156
+ return c;
157
+}
158
+
159
+template<>
160
+inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b) {
161
+ __nv_bfloat162 s = bf162bf162(a);
162
+ bf16_8_t c;
163
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
164
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
165
+ c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.z);
166
+ c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.w);
167
+ return c;
168
+}
169
+
170
+template<>
171
+inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b) {
172
+ float fa = __bfloat162float(a);
173
+ float fb = __bfloat162float(b);
174
+ return fa * fb;
175
+}
176
+
177
+template<>
178
+inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b) {
179
+ float2 fa = bf1622float2(a);
180
+ float2 fb = bf1622float2(b);
181
+ return mul<float2, float2, float2>(fa, fb);
182
+}
183
+
184
+template<>
185
+inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b) {
186
+ return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
187
+}
188
+
189
+template<>
190
+inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b) {
191
+ Float4_ fc;
192
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
193
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
194
+ return fc;
195
+}
196
+
197
+template<>
198
+inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b) {
199
+ __nv_bfloat162 s = bf162bf162(a);
200
+ Float4_ fc;
201
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
202
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
203
+ return fc;
204
+}
205
+
206
+template<>
207
+inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b) {
208
+ Float8_ fc;
209
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
210
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
211
+ fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
212
+ fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
213
+ return fc;
214
+}
215
+
216
+template<>
217
+inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b) {
218
+ __nv_bfloat162 s = bf162bf162(a);
219
+ Float8_ fc;
220
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
221
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
222
+ fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.z);
223
+ fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.w);
224
+ return fc;
225
+}
226
+
227
+// Vector fused multiply-add.
228
+inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c) {
229
+ return __hfma2(a, b, c);
230
+}
231
+
232
+inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c) {
233
+ return __hfma2(bf162bf162(a), b, c);
234
+}
235
+
236
+inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c) {
237
+ bf16_4_t d;
238
+ d.x = fma(a.x, b.x, c.x);
239
+ d.y = fma(a.y, b.y, c.y);
240
+ return d;
241
+}
242
+
243
+inline __device__ bf16_4_t fma(__nv_bfloat16 a, bf16_4_t b, bf16_4_t c) {
244
+ __nv_bfloat162 s = bf162bf162(a);
245
+ bf16_4_t d;
246
+ d.x = fma(s, b.x, c.x);
247
+ d.y = fma(s, b.y, c.y);
248
+ return d;
249
+}
250
+
251
+inline __device__ bf16_8_t fma(bf16_8_t a, bf16_8_t b, bf16_8_t c) {
252
+ bf16_8_t d;
253
+ d.x = fma(a.x, b.x, c.x);
254
+ d.y = fma(a.y, b.y, c.y);
255
+ d.z = fma(a.z, b.z, c.z);
256
+ d.w = fma(a.w, b.w, c.w);
257
+ return d;
258
+}
259
+
260
+inline __device__ bf16_8_t fma(__nv_bfloat16 a, bf16_8_t b, bf16_8_t c) {
261
+ __nv_bfloat162 s = bf162bf162(a);
262
+ bf16_8_t d;
263
+ d.x = fma(s, b.x, c.x);
264
+ d.y = fma(s, b.y, c.y);
265
+ d.z = fma(s, b.z, c.z);
266
+ d.w = fma(s, b.w, c.w);
267
+ return d;
268
+}
269
+
270
+inline __device__ float fma(__nv_bfloat16 a, __nv_bfloat16 b, float fc) {
271
+ return __bfloat162float(a) * __bfloat162float(b) + fc;
272
+}
273
+
274
+inline __device__ float2 fma(__nv_bfloat162 a, __nv_bfloat162 b, float2 fc) {
275
+ float2 fa = bf1622float2(a);
276
+ float2 fb = bf1622float2(b);
277
+ return fma(fa, fb, fc);
278
+}
279
+
280
+inline __device__ float2 fma(__nv_bfloat16 a, __nv_bfloat162 b, float2 fc) {
281
+ return fma(bf162bf162(a), b, fc);
282
+}
283
+
284
+inline __device__ Float4_ fma(bf16_4_t a, bf16_4_t b, Float4_ fc) {
285
+ Float4_ fd;
286
+ fd.x = fma(a.x, b.x, fc.x);
287
+ fd.y = fma(a.y, b.y, fc.y);
288
+ return fd;
289
+}
290
+
291
+inline __device__ Float4_ fma(__nv_bfloat16 a, bf16_4_t b, Float4_ fc) {
292
+ __nv_bfloat162 s = bf162bf162(a);
293
+ Float4_ fd;
294
+ fd.x = fma(s, b.x, fc.x);
295
+ fd.y = fma(s, b.y, fc.y);
296
+ return fd;
297
+}
298
+
299
+inline __device__ Float8_ fma(bf16_8_t a, bf16_8_t b, Float8_ fc) {
300
+ Float8_ fd;
301
+ fd.x = fma(a.x, b.x, fc.x);
302
+ fd.y = fma(a.y, b.y, fc.y);
303
+ fd.z = fma(a.z, b.z, fc.z);
304
+ fd.w = fma(a.w, b.w, fc.w);
305
+ return fd;
306
+}
307
+
308
+inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc) {
309
+ __nv_bfloat162 s = bf162bf162(a);
310
+ Float8_ fd;
311
+ fd.x = fma(s, b.x, fc.x);
312
+ fd.y = fma(s, b.y, fc.y);
313
+ fd.z = fma(s, b.z, fc.z);
314
+ fd.w = fma(s, b.w, fc.w);
315
+ return fd;
316
+}
317
+
318
+// Vector sum.
319
+template<>
320
+inline __device__ float sum(__nv_bfloat16 v) {
321
+ return __bfloat162float(v);
322
+}
323
+
324
+template<>
325
+inline __device__ float sum(__nv_bfloat162 v) {
326
+ float2 vf = bf1622float2(v);
327
+ return vf.x + vf.y;
328
+}
329
+
330
+template<>
331
+inline __device__ float sum(bf16_4_t v) {
332
+ return sum(v.x) + sum(v.y);
333
+}
334
+
335
+template<>
336
+inline __device__ float sum(bf16_8_t v) {
337
+ return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
338
+}
339
+
340
+// From float32 to bfloat16.
341
+inline __device__ void from_float(__nv_bfloat16& dst, float src) {
342
+ dst = __float2bfloat16(src);
343
+}
344
+
345
+inline __device__ void from_float(__nv_bfloat162& dst, float2 src) {
346
+ dst = __float22bfloat162_rn(src);
347
+}
348
+
349
+inline __device__ void from_float(bf16_4_t& dst, Float4_ src) {
350
+ dst.x = __float22bfloat162_rn(src.x);
351
+ dst.y = __float22bfloat162_rn(src.y);
352
+}
353
+
354
+inline __device__ void from_float(bf16_8_t& dst, Float8_ src) {
355
+ dst.x = __float22bfloat162_rn(src.x);
356
+ dst.y = __float22bfloat162_rn(src.y);
357
+ dst.z = __float22bfloat162_rn(src.z);
358
+ dst.w = __float22bfloat162_rn(src.w);
359
+}
360
+
361
+} // namespace cacheflow
csrc/attention/dtype_float32.cuh
+1
-1
@@ -6,7 +6,7 @@
6
7
namespace cacheflow {
8
9
-// Define FP32 vector data types.
9
+// Define custom FP32 vector data types.
10
struct Float4_ {
11
float2 x;
12
float2 y;
csrc/cache_kernels.cu
+55
-43
@@ -14,14 +14,16 @@ void swap_blocks(
14
torch::Device dst_device = dst.device();
15
cudaMemcpyKind memcpy_type;
16
if (src_device.is_cuda() && dst_device.is_cuda()) {
17
- assert(src_device.index() == dst_device.index());
17
+ TORCH_CHECK(
18
+ src_device.index() == dst_device.index(),
19
+ "src and dst must be on the same GPU");
20
memcpy_type = cudaMemcpyDeviceToDevice;
21
} else if (src_device.is_cuda() && dst_device.is_cpu()) {
22
memcpy_type = cudaMemcpyDeviceToHost;
23
} else if (src_device.is_cpu() && dst_device.is_cuda()) {
24
memcpy_type = cudaMemcpyHostToDevice;
25
} else {
24
- assert(false);
26
+ TORCH_CHECK(false, "Invalid device combination");
27
}
28
29
void *src_ptr = src.data_ptr();
@@ -29,6 +31,7 @@ void swap_blocks(
31
32
const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
33
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
34
+ // NOTE(woosuk): This can be slow if the number of blocks is large.
35
for (const auto& pair : block_mapping) {
36
int64_t src_block_number = pair.first;
37
int64_t dst_block_number = pair.second;
@@ -122,7 +125,9 @@ void copy_blocks(
125
dim3 grid(num_layers, num_pairs);
126
dim3 block(std::min(1024, numel_per_block));
127
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
125
- AT_DISPATCH_FLOATING_TYPES_AND_HALF(
128
+ AT_DISPATCH_FLOATING_TYPES_AND2(
129
+ at::ScalarType::Half,
130
+ at::ScalarType::BFloat16,
131
key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
132
cacheflow::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
133
key_cache_ptrs_tensor.data_ptr<int64_t>(),
@@ -176,6 +181,50 @@ __global__ void reshape_and_cache_kernel(
181
}
182
}
183
184
+} // namespace cacheflow
185
+
186
+void reshape_and_cache(
187
+ torch::Tensor& key, // [num_tokens, num_heads, head_size]
188
+ torch::Tensor& value, // [num_tokens, num_heads, head_size]
189
+ torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
190
+ torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
191
+ torch::Tensor& slot_mapping) // [num_tokens]
192
+{
193
+ int num_tokens = key.size(0);
194
+ int num_heads = key.size(1);
195
+ int head_size = key.size(2);
196
+ int block_size = key_cache.size(3);
197
+ int x = key_cache.size(4);
198
+
199
+ int key_stride = key.stride(0);
200
+ int value_stride = value.stride(0);
201
+
202
+ dim3 grid(num_tokens);
203
+ dim3 block(std::min(num_heads * head_size, 512));
204
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
205
+ AT_DISPATCH_FLOATING_TYPES_AND2(
206
+ at::ScalarType::Half,
207
+ at::ScalarType::BFloat16,
208
+ key.scalar_type(),
209
+ "reshape_and_cache_kernel",
210
+ [&] {
211
+ cacheflow::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
212
+ key.data_ptr<scalar_t>(),
213
+ value.data_ptr<scalar_t>(),
214
+ key_cache.data_ptr<scalar_t>(),
215
+ value_cache.data_ptr<scalar_t>(),
216
+ slot_mapping.data_ptr<int>(),
217
+ key_stride,
218
+ value_stride,
219
+ num_heads,
220
+ head_size,
221
+ block_size,
222
+ x);
223
+ });
224
+}
225
+
226
+namespace cacheflow {
227
+
228
// Grid: (num_blocks, block_size).
229
template<typename scalar_t>
230
__global__ void gather_cached_kv_kernel(
@@ -296,45 +345,6 @@ __global__ void gather_cached_kv_kernel_optimized(
345
346
} // namespace cacheflow
347
299
-void reshape_and_cache(
300
- torch::Tensor& key, // [num_tokens, num_heads, head_size]
301
- torch::Tensor& value, // [num_tokens, num_heads, head_size]
302
- torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
303
- torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
304
- torch::Tensor& slot_mapping) // [num_tokens]
305
-{
306
- int num_tokens = key.size(0);
307
- int num_heads = key.size(1);
308
- int head_size = key.size(2);
309
- int block_size = key_cache.size(3);
310
- int x = key_cache.size(4);
311
-
312
- int key_stride = key.stride(0);
313
- int value_stride = value.stride(0);
314
-
315
- dim3 grid(num_tokens);
316
- dim3 block(std::min(num_heads * head_size, 512));
317
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
318
- AT_DISPATCH_FLOATING_TYPES_AND_HALF(
319
- key.scalar_type(),
320
- "reshape_and_cache_kernel",
321
- [&] {
322
- cacheflow::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
323
- key.data_ptr<scalar_t>(),
324
- value.data_ptr<scalar_t>(),
325
- key_cache.data_ptr<scalar_t>(),
326
- value_cache.data_ptr<scalar_t>(),
327
- slot_mapping.data_ptr<int>(),
328
- key_stride,
329
- value_stride,
330
- num_heads,
331
- head_size,
332
- block_size,
333
- x);
334
- });
335
-}
336
-
337
-
348
void gather_cached_kv(
349
torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
350
torch::Tensor& value, // [out] [num_tokens, num_heads, head_size]
@@ -354,7 +364,9 @@ void gather_cached_kv(
364
dim3 grid(num_tokens);
365
dim3 block(std::min(num_heads * head_size, 512));
366
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
357
- AT_DISPATCH_FLOATING_TYPES_AND_HALF(
367
+ AT_DISPATCH_FLOATING_TYPES_AND2(
368
+ at::ScalarType::Half,
369
+ at::ScalarType::BFloat16,
370
key.scalar_type(),
371
"gather_cached_kv_kernel_optimized",
372
[&] {
csrc/layernorm_kernels.cu
+3
-1
@@ -46,7 +46,9 @@ void rms_norm(
46
dim3 grid(num_tokens);
47
dim3 block(std::min(hidden_size, 1024));
48
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
49
- AT_DISPATCH_FLOATING_TYPES_AND_HALF(
49
+ AT_DISPATCH_FLOATING_TYPES_AND2(
50
+ at::ScalarType::Half,
51
+ at::ScalarType::BFloat16,
52
input.scalar_type(),
53
"rms_norm_kernel",
54
[&] {
csrc/pos_encoding_kernels.cu
+3
-1
@@ -64,7 +64,9 @@ void rotary_embedding_neox(
64
dim3 grid(num_tokens);
65
dim3 block(std::min(num_heads * rot_dim / 2, 512));
66
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
67
- AT_DISPATCH_FLOATING_TYPES_AND_HALF(
67
+ AT_DISPATCH_FLOATING_TYPES_AND2(
68
+ at::ScalarType::Half,
69
+ at::ScalarType::BFloat16,
70
query.scalar_type(),
71
"rotary_embedding_neox",
72
[&] {
setup.py
+15
-1
@@ -1,9 +1,22 @@
1
import setuptools
2
+import torch
3
from torch.utils import cpp_extension
4
5
CXX_FLAGS = ['-g']
6
NVCC_FLAGS = ['-O2']
7
8
+if not torch.cuda.is_available():
9
+ raise RuntimeError(
10
+ f'Cannot find CUDA at CUDA_HOME: {cpp_extension.CUDA_HOME}. '
11
+ 'CUDA must be available in order to build the package.')
12
+
13
+# FIXME(woosuk): Consider the case where the machine has multiple GPUs with
14
+# different compute capabilities.
15
+compute_capability = torch.cuda.get_device_capability()
16
+major, minor = compute_capability
17
+# Enable bfloat16 support if the compute capability is >= 8.0.
18
+if major >= 8:
19
+ NVCC_FLAGS.append('-DENABLE_BF16')
20
21
ext_modules = []
22
@@ -23,7 +36,7 @@ attention_extension = cpp_extension.CUDAExtension(
36
)
37
ext_modules.append(attention_extension)
38
26
-# Positional encodings.
39
+# Positional encoding kernels.
40
positional_encoding_extension = cpp_extension.CUDAExtension(
41
name='cacheflow.pos_encoding_ops',
42
sources=['csrc/pos_encoding.cpp', 'csrc/pos_encoding_kernels.cu'],
@@ -39,6 +52,7 @@ layernorm_extension = cpp_extension.CUDAExtension(
52
)
53
ext_modules.append(layernorm_extension)
54
55
+# Activation kernels.
56
activation_extension = cpp_extension.CUDAExtension(
57
name='cacheflow.activation_ops',
58
sources=['csrc/activation.cpp', 'csrc/activation_kernels.cu'],