A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Support FP8-E5M2 KV Cache (#2279)
Co-authored-by: zhaoyang <zhao.yang16@zte.com.cn> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
zhaoyang-star committed
Jan 29, 2024 at 08:43 UTC
9090bf02e74334a8020b454814e0d00fa780fd79
26 files changed
+912
-196
benchmarks/benchmark_latency.py
+8
@@ -24,6 +24,7 @@ def main(args: argparse.Namespace):
24
trust_remote_code=args.trust_remote_code,
25
dtype=args.dtype,
26
enforce_eager=args.enforce_eager,
27
+ kv_cache_dtype=args.kv_cache_dtype,
28
)
29
30
sampling_params = SamplingParams(
@@ -117,6 +118,13 @@ if __name__ == '__main__':
118
parser.add_argument('--enforce-eager',
119
action='store_true',
120
help='enforce eager mode and disable CUDA graph')
121
+ parser.add_argument(
122
+ "--kv-cache-dtype",
123
+ type=str,
124
+ choices=['auto', 'fp8_e5m2'],
125
+ default='auto',
126
+ help=
127
+ 'Data type for kv cache storage. If "auto", will use model data type.')
128
parser.add_argument(
129
'--profile',
130
action='store_true',
benchmarks/benchmark_throughput.py
+11
-1
@@ -71,6 +71,7 @@ def run_vllm(
71
dtype: str,
72
max_model_len: Optional[int],
73
enforce_eager: bool,
74
+ kv_cache_dtype: str,
75
) -> float:
76
from vllm import LLM, SamplingParams
77
llm = LLM(
@@ -83,6 +84,7 @@ def run_vllm(
84
dtype=dtype,
85
max_model_len=max_model_len,
86
enforce_eager=enforce_eager,
87
+ kv_cache_dtype=kv_cache_dtype,
88
)
89
90
# Add the requests to the engine.
@@ -206,7 +208,8 @@ def main(args: argparse.Namespace):
208
args.quantization, args.tensor_parallel_size,
209
args.seed, args.n, args.use_beam_search,
210
args.trust_remote_code, args.dtype,
209
- args.max_model_len, args.enforce_eager)
211
+ args.max_model_len, args.enforce_eager,
212
+ args.kv_cache_dtype)
213
elif args.backend == "hf":
214
assert args.tensor_parallel_size == 1
215
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -284,6 +287,13 @@ if __name__ == "__main__":
287
parser.add_argument("--enforce-eager",
288
action="store_true",
289
help="enforce eager execution")
290
+ parser.add_argument(
291
+ "--kv-cache-dtype",
292
+ type=str,
293
+ choices=["auto", "fp8_e5m2"],
294
+ default="auto",
295
+ help=
296
+ 'Data type for kv cache storage. If "auto", will use model data type.')
297
args = parser.parse_args()
298
if args.tokenizer is None:
299
args.tokenizer = args.model
benchmarks/kernels/benchmark_paged_attention.py
+18
-15
@@ -1,9 +1,11 @@
1
+from typing import Optional
2
import argparse
3
import random
4
import time
5
6
import torch
7
8
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
9
from vllm._C import ops
10
11
NUM_BLOCKS = 1024
@@ -23,6 +25,7 @@ def main(
25
dtype: torch.dtype,
26
seed: int,
27
do_profile: bool,
28
+ kv_cache_dtype: Optional[str] = None,
29
) -> None:
30
random.seed(seed)
31
torch.random.manual_seed(seed)
@@ -59,15 +62,10 @@ def main(
62
block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
63
64
# Create the KV cache.
62
- x = 16 // torch.tensor([], dtype=dtype).element_size()
63
- key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x, block_size, x)
64
- key_cache = torch.empty(size=key_cache_shape, dtype=dtype, device="cuda")
65
- key_cache.uniform_(-scale, scale)
66
- value_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size, block_size)
67
- value_cache = torch.empty(size=value_cache_shape,
68
- dtype=dtype,
69
- device="cuda")
70
- value_cache.uniform_(-scale, scale)
65
+ key_caches, value_caches = create_kv_caches_with_random(
66
+ NUM_BLOCKS, block_size, 1, num_kv_heads, head_size, kv_cache_dtype,
67
+ dtype)
68
+ key_cache, value_cache = key_caches[0], value_caches[0]
69
70
# Prepare for the paged attention kernel.
71
output = torch.empty_like(query)
@@ -106,6 +104,7 @@ def main(
104
block_size,
105
max_context_len,
106
alibi_slopes,
107
+ kv_cache_dtype,
108
)
109
elif version == "v2":
110
ops.paged_attention_v2(
@@ -123,6 +122,7 @@ def main(
122
block_size,
123
max_context_len,
124
alibi_slopes,
125
+ kv_cache_dtype,
126
)
127
else:
128
raise ValueError(f"Invalid version: {version}")
@@ -168,16 +168,18 @@ if __name__ == '__main__':
168
default="half")
169
parser.add_argument("--seed", type=int, default=0)
170
parser.add_argument("--profile", action="store_true")
171
+ parser.add_argument(
172
+ "--kv-cache-dtype",
173
+ type=str,
174
+ choices=["auto", "fp8_e5m2"],
175
+ default="auto",
176
+ help=
177
+ 'Data type for kv cache storage. If "auto", will use model data type.')
178
args = parser.parse_args()
179
print(args)
180
181
if args.num_query_heads % args.num_kv_heads != 0:
182
raise ValueError("num_query_heads must be divisible by num_kv_heads")
176
- dtype_to_torch_dtype = {
177
- "half": torch.half,
178
- "bfloat16": torch.bfloat16,
179
- "float": torch.float,
180
- }
183
main(
184
version=args.version,
185
num_seqs=args.batch_size,
@@ -187,7 +189,8 @@ if __name__ == '__main__':
189
head_size=args.head_size,
190
block_size=args.block_size,
191
use_alibi=args.use_alibi,
190
- dtype=dtype_to_torch_dtype[args.dtype],
192
+ dtype=STR_DTYPE_TO_TORCH_DTYPE[args.dtype],
193
seed=args.seed,
194
do_profile=args.profile,
195
+ kv_cache_dtype=args.kv_cache_dtype,
196
)
csrc/attention/attention_dtypes.h
+1
@@ -4,3 +4,4 @@
4
#include "dtype_float16.cuh"
5
#include "dtype_float32.cuh"
6
#include "dtype_bfloat16.cuh"
7
+#include "dtype_fp8_e5m2.cuh"
csrc/attention/attention_kernels.cu
+164
-95
@@ -25,6 +25,7 @@
25
26
#include "attention_dtypes.h"
27
#include "attention_utils.cuh"
28
+#include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
29
30
#include <algorithm>
31
@@ -79,17 +80,19 @@ inline __device__ float block_sum(float* red_smem, float sum) {
80
// Grid: (num_heads, num_seqs, max_num_partitions).
81
template<
82
typename scalar_t,
83
+ typename cache_t,
84
int HEAD_SIZE,
85
int BLOCK_SIZE,
86
int NUM_THREADS,
87
+ bool IS_FP8_E5M2_KV_CACHE,
88
int PARTITION_SIZE = 0> // Zero means no partitioning.
89
__device__ void paged_attention_kernel(
90
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
91
float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
92
scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
93
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
91
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
92
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
94
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
95
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
96
const int num_kv_heads, // [num_heads]
97
const float scale,
98
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
@@ -145,6 +148,9 @@ __device__ void paged_attention_kernel(
148
constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
149
using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
150
using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
151
+#ifdef ENABLE_FP8_E5M2
152
+ using Quant_vec = typename Vec<cache_t, VEC_SIZE>::Type;
153
+#endif
154
155
constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
156
constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
@@ -176,7 +182,7 @@ __device__ void paged_attention_kernel(
182
183
// x == THREAD_GROUP_SIZE * VEC_SIZE
184
// Each thread group fetches x elements from the key at a time.
179
- constexpr int x = 16 / sizeof(scalar_t);
185
+ constexpr int x = 16 / sizeof(cache_t);
186
float qk_max = -FLT_MAX;
187
188
// Iterate over the key blocks.
@@ -202,13 +208,23 @@ __device__ void paged_attention_kernel(
208
209
#pragma unroll
210
for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
205
- const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride
206
- + kv_head_idx * kv_head_stride
207
- + physical_block_offset * x;
211
+ const cache_t* k_ptr = k_cache + physical_block_number * kv_block_stride
212
+ + kv_head_idx * kv_head_stride
213
+ + physical_block_offset * x;
214
const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
215
const int offset1 = (vec_idx * VEC_SIZE) / x;
216
const int offset2 = (vec_idx * VEC_SIZE) % x;
211
- k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
217
+ if constexpr (IS_FP8_E5M2_KV_CACHE) {
218
+#ifdef ENABLE_FP8_E5M2
219
+ Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
220
+ // Vector conversion from Quant_vec to K_vec.
221
+ k_vecs[j] = fp8_e5m2_unscaled::vec_conversion<K_vec, Quant_vec>(k_vec_quant);
222
+#else
223
+ assert(false);
224
+#endif
225
+ } else {
226
+ k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
227
+ }
228
}
229
230
// Compute dot product.
@@ -282,6 +298,9 @@ __device__ void paged_attention_kernel(
298
constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
299
using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
300
using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
301
+#ifdef ENABLE_FP8_E5M2
302
+ using V_quant_vec = typename Vec<cache_t, V_VEC_SIZE>::Type;
303
+#endif
304
using Float_L_vec = typename FloatVec<L_vec>::Type;
305
306
constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
@@ -307,14 +326,25 @@ __device__ void paged_attention_kernel(
326
L_vec logits_vec;
327
from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx - start_token_idx));
328
310
- const scalar_t* v_ptr = v_cache + physical_block_number * kv_block_stride
311
- + kv_head_idx * kv_head_stride;
329
+ const cache_t* v_ptr = v_cache + physical_block_number * kv_block_stride
330
+ + kv_head_idx * kv_head_stride;
331
#pragma unroll
332
for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
333
const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
334
if (row_idx < HEAD_SIZE) {
335
const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
317
- V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
336
+ V_vec v_vec;
337
+ if constexpr (IS_FP8_E5M2_KV_CACHE) {
338
+#ifdef ENABLE_FP8_E5M2
339
+ V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
340
+ // Vector conversion from V_quant_vec to V_vec.
341
+ v_vec = fp8_e5m2_unscaled::vec_conversion<V_vec, V_quant_vec>(v_quant_vec);
342
+#else
343
+ assert(false);
344
+#endif
345
+ } else {
346
+ v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
347
+ }
348
if (block_idx == num_context_blocks - 1) {
349
// NOTE(woosuk): When v_vec contains the tokens that are out of the context,
350
// we should explicitly zero out the values since they may contain NaNs.
@@ -395,14 +425,16 @@ __device__ void paged_attention_kernel(
425
// Grid: (num_heads, num_seqs, 1).
426
template<
427
typename scalar_t,
428
+ typename cache_t,
429
int HEAD_SIZE,
430
int BLOCK_SIZE,
400
- int NUM_THREADS>
431
+ int NUM_THREADS,
432
+ bool IS_FP8_E5M2_KV_CACHE>
433
__global__ void paged_attention_v1_kernel(
434
scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
435
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
404
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
405
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
436
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
437
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
438
const int num_kv_heads, // [num_heads]
439
const float scale,
440
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
@@ -412,7 +444,7 @@ __global__ void paged_attention_v1_kernel(
444
const int q_stride,
445
const int kv_block_stride,
446
const int kv_head_stride) {
415
- paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
447
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE>(
448
/* exp_sums */ nullptr, /* max_logits */ nullptr,
449
out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
450
max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
@@ -421,17 +453,19 @@ __global__ void paged_attention_v1_kernel(
453
// Grid: (num_heads, num_seqs, max_num_partitions).
454
template<
455
typename scalar_t,
456
+ typename cache_t,
457
int HEAD_SIZE,
458
int BLOCK_SIZE,
459
int NUM_THREADS,
460
+ bool IS_FP8_E5M2_KV_CACHE,
461
int PARTITION_SIZE>
462
__global__ void paged_attention_v2_kernel(
463
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
464
float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
465
scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
466
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
433
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
434
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
467
+ const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
468
+ const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
469
const int num_kv_heads, // [num_heads]
470
const float scale,
471
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
@@ -441,7 +475,7 @@ __global__ void paged_attention_v2_kernel(
475
const int q_stride,
476
const int kv_block_stride,
477
const int kv_head_stride) {
444
- paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE>(
478
+ paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE>(
479
exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
480
block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
481
q_stride, kv_block_stride, kv_head_stride);
@@ -550,10 +584,10 @@ __global__ void paged_attention_v2_reduce_kernel(
584
585
#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
586
VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
553
- ((void*)vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>), \
554
- shared_mem_size); \
555
- vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
556
- <<<grid, block, shared_mem_size, stream>>>( \
587
+ ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
588
+ IS_FP8_E5M2_KV_CACHE>), shared_mem_size); \
589
+ vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
590
+ IS_FP8_E5M2_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
591
out_ptr, \
592
query_ptr, \
593
key_cache_ptr, \
@@ -571,7 +605,9 @@ __global__ void paged_attention_v2_reduce_kernel(
605
// TODO(woosuk): Tune NUM_THREADS.
606
template<
607
typename T,
608
+ typename CACHE_T,
609
int BLOCK_SIZE,
610
+ bool IS_FP8_E5M2_KV_CACHE,
611
int NUM_THREADS = 128>
612
void paged_attention_v1_launcher(
613
torch::Tensor& out,
@@ -602,8 +638,8 @@ void paged_attention_v1_launcher(
638
639
T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
640
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
605
- T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
606
- T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
641
+ CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
642
+ CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
643
int* block_tables_ptr = block_tables.data_ptr<int>();
644
int* context_lens_ptr = context_lens.data_ptr<int>();
645
@@ -647,35 +683,35 @@ void paged_attention_v1_launcher(
683
}
684
}
685
650
-#define CALL_V1_LAUNCHER(T, BLOCK_SIZE) \
651
- paged_attention_v1_launcher<T, BLOCK_SIZE>( \
652
- out, \
653
- query, \
654
- key_cache, \
655
- value_cache, \
656
- num_kv_heads, \
657
- scale, \
658
- block_tables, \
659
- context_lens, \
660
- max_context_len, \
686
+#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
687
+ paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
688
+ out, \
689
+ query, \
690
+ key_cache, \
691
+ value_cache, \
692
+ num_kv_heads, \
693
+ scale, \
694
+ block_tables, \
695
+ context_lens, \
696
+ max_context_len, \
697
alibi_slopes);
698
699
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
700
// 1, 2, 4, 64, 128, 256.
665
-#define CALL_V1_LAUNCHER_BLOCK_SIZE(T) \
666
- switch (block_size) { \
667
- case 8: \
668
- CALL_V1_LAUNCHER(T, 8); \
669
- break; \
670
- case 16: \
671
- CALL_V1_LAUNCHER(T, 16); \
672
- break; \
673
- case 32: \
674
- CALL_V1_LAUNCHER(T, 32); \
675
- break; \
676
- default: \
677
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
678
- break; \
701
+#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
702
+ switch (block_size) { \
703
+ case 8: \
704
+ CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
705
+ break; \
706
+ case 16: \
707
+ CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
708
+ break; \
709
+ case 32: \
710
+ CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
711
+ break; \
712
+ default: \
713
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
714
+ break; \
715
}
716
717
void paged_attention_v1(
@@ -689,20 +725,36 @@ void paged_attention_v1(
725
torch::Tensor& context_lens, // [num_seqs]
726
int block_size,
727
int max_context_len,
692
- const c10::optional<torch::Tensor>& alibi_slopes) {
693
- if (query.dtype() == at::ScalarType::Float) {
694
- CALL_V1_LAUNCHER_BLOCK_SIZE(float);
695
- } else if (query.dtype() == at::ScalarType::Half) {
696
- CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t);
697
- } else if (query.dtype() == at::ScalarType::BFloat16) {
698
- CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
728
+ const c10::optional<torch::Tensor>& alibi_slopes,
729
+ const std::string& kv_cache_dtype) {
730
+ if (kv_cache_dtype == "auto") {
731
+ if (query.dtype() == at::ScalarType::Float) {
732
+ CALL_V1_LAUNCHER_BLOCK_SIZE(float, float, false);
733
+ } else if (query.dtype() == at::ScalarType::Half) {
734
+ CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
735
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
736
+ CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
737
+ } else {
738
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
739
+ }
740
+ } else if (kv_cache_dtype == "fp8_e5m2") {
741
+ if (query.dtype() == at::ScalarType::Float) {
742
+ CALL_V1_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
743
+ } else if (query.dtype() == at::ScalarType::Half) {
744
+ CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
745
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
746
+ CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
747
+ } else {
748
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
749
+ }
750
} else {
700
- TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
751
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
752
}
753
}
754
755
#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
705
- vllm::paged_attention_v2_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE> \
756
+ vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
757
+ IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE> \
758
<<<grid, block, shared_mem_size, stream>>>( \
759
exp_sums_ptr, \
760
max_logits_ptr, \
@@ -730,7 +782,9 @@ void paged_attention_v1(
782
783
template<
784
typename T,
785
+ typename CACHE_T,
786
int BLOCK_SIZE,
787
+ bool IS_FP8_E5M2_KV_CACHE,
788
int NUM_THREADS = 128,
789
int PARTITION_SIZE = 512>
790
void paged_attention_v2_launcher(
@@ -768,8 +822,8 @@ void paged_attention_v2_launcher(
822
float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
823
T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
824
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
771
- T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
772
- T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
825
+ CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
826
+ CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
827
int* block_tables_ptr = block_tables.data_ptr<int>();
828
int* context_lens_ptr = context_lens.data_ptr<int>();
829
@@ -816,38 +870,38 @@ void paged_attention_v2_launcher(
870
}
871
}
872
819
-#define CALL_V2_LAUNCHER(T, BLOCK_SIZE) \
820
- paged_attention_v2_launcher<T, BLOCK_SIZE>( \
821
- out, \
822
- exp_sums, \
823
- max_logits, \
824
- tmp_out, \
825
- query, \
826
- key_cache, \
827
- value_cache, \
828
- num_kv_heads, \
829
- scale, \
830
- block_tables, \
831
- context_lens, \
832
- max_context_len, \
873
+#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
874
+ paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
875
+ out, \
876
+ exp_sums, \
877
+ max_logits, \
878
+ tmp_out, \
879
+ query, \
880
+ key_cache, \
881
+ value_cache, \
882
+ num_kv_heads, \
883
+ scale, \
884
+ block_tables, \
885
+ context_lens, \
886
+ max_context_len, \
887
alibi_slopes);
888
889
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
890
// 1, 2, 4, 64, 128, 256.
837
-#define CALL_V2_LAUNCHER_BLOCK_SIZE(T) \
838
- switch (block_size) { \
839
- case 8: \
840
- CALL_V2_LAUNCHER(T, 8); \
841
- break; \
842
- case 16: \
843
- CALL_V2_LAUNCHER(T, 16); \
844
- break; \
845
- case 32: \
846
- CALL_V2_LAUNCHER(T, 32); \
847
- break; \
848
- default: \
849
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
850
- break; \
891
+#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
892
+ switch (block_size) { \
893
+ case 8: \
894
+ CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
895
+ break; \
896
+ case 16: \
897
+ CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
898
+ break; \
899
+ case 32: \
900
+ CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
901
+ break; \
902
+ default: \
903
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
904
+ break; \
905
}
906
907
void paged_attention_v2(
@@ -864,15 +918,30 @@ void paged_attention_v2(
918
torch::Tensor& context_lens, // [num_seqs]
919
int block_size,
920
int max_context_len,
867
- const c10::optional<torch::Tensor>& alibi_slopes) {
868
- if (query.dtype() == at::ScalarType::Float) {
869
- CALL_V2_LAUNCHER_BLOCK_SIZE(float);
870
- } else if (query.dtype() == at::ScalarType::Half) {
871
- CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t);
872
- } else if (query.dtype() == at::ScalarType::BFloat16) {
873
- CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
921
+ const c10::optional<torch::Tensor>& alibi_slopes,
922
+ const std::string& kv_cache_dtype) {
923
+ if (kv_cache_dtype == "auto") {
924
+ if (query.dtype() == at::ScalarType::Float) {
925
+ CALL_V2_LAUNCHER_BLOCK_SIZE(float, float, false);
926
+ } else if (query.dtype() == at::ScalarType::Half) {
927
+ CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
928
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
929
+ CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
930
+ } else {
931
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
932
+ }
933
+ } else if (kv_cache_dtype == "fp8_e5m2") {
934
+ if (query.dtype() == at::ScalarType::Float) {
935
+ CALL_V2_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
936
+ } else if (query.dtype() == at::ScalarType::Half) {
937
+ CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
938
+ } else if (query.dtype() == at::ScalarType::BFloat16) {
939
+ CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
940
+ } else {
941
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
942
+ }
943
} else {
875
- TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
944
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
945
}
946
}
947
csrc/attention/dtype_fp8_e5m2.cuh
new
+35
@@ -0,0 +1,35 @@
1
+#pragma once
2
+
3
+#include "attention_generic.cuh"
4
+
5
+#include <stdint.h>
6
+#ifdef ENABLE_FP8_E5M2
7
+#include <cuda_fp8.h>
8
+#endif
9
+
10
+namespace vllm {
11
+#ifdef ENABLE_FP8_E5M2
12
+// fp8 vector types for quantization of kv cache
13
+
14
+template<>
15
+struct Vec<uint8_t, 1> {
16
+ using Type = uint8_t;
17
+};
18
+
19
+template<>
20
+struct Vec<uint8_t, 2> {
21
+ using Type = uint16_t;
22
+};
23
+
24
+template<>
25
+struct Vec<uint8_t, 4> {
26
+ using Type = uint32_t;
27
+};
28
+
29
+template<>
30
+struct Vec<uint8_t, 8> {
31
+ using Type = uint2;
32
+};
33
+#endif // ENABLE_FP8_E5M2
34
+
35
+} // namespace vllm
csrc/cache.h
+7
-1
@@ -20,7 +20,8 @@ void reshape_and_cache(
20
torch::Tensor& value,
21
torch::Tensor& key_cache,
22
torch::Tensor& value_cache,
23
- torch::Tensor& slot_mapping);
23
+ torch::Tensor& slot_mapping,
24
+ const std::string& kv_cache_dtype);
25
26
void gather_cached_kv(
27
torch::Tensor& key,
@@ -28,3 +29,8 @@ void gather_cached_kv(
29
torch::Tensor& key_cache,
30
torch::Tensor& value_cache,
31
torch::Tensor& slot_mapping);
32
+
33
+// Just for unittest
34
+void convert_fp8_e5m2(
35
+ torch::Tensor& src_cache,
36
+ torch::Tensor& dst_cache);
csrc/cache_kernels.cu
+108
-27
@@ -4,6 +4,7 @@
4
5
#include "cuda_compat.h"
6
#include "dispatch_utils.h"
7
+#include "quantization/fp8_e5m2_kvcache/quant_utils.cuh"
8
9
#include <algorithm>
10
#include <cassert>
@@ -131,7 +132,7 @@ void copy_blocks(
132
dim3 block(std::min(1024, numel_per_block));
133
const at::cuda::OptionalCUDAGuard device_guard(cache_device);
134
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
134
- VLLM_DISPATCH_FLOATING_TYPES(
135
+ VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
136
key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
137
vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
138
key_cache_ptrs_tensor.data_ptr<int64_t>(),
@@ -143,12 +144,12 @@ void copy_blocks(
144
145
namespace vllm {
146
146
-template<typename scalar_t>
147
+template<typename scalar_t, typename cache_t, bool is_fp8_e5m2_kv_cache>
148
__global__ void reshape_and_cache_kernel(
149
const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
150
const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
150
- scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
151
- scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
151
+ cache_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
152
+ cache_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
153
const int64_t* __restrict__ slot_mapping, // [num_tokens]
154
const int key_stride,
155
const int value_stride,
@@ -185,19 +186,45 @@ __global__ void reshape_and_cache_kernel(
186
+ head_idx * head_size * block_size
187
+ head_offset * block_size
188
+ block_offset;
188
- key_cache[tgt_key_idx] = key[src_key_idx];
189
- value_cache[tgt_value_idx] = value[src_value_idx];
189
+ scalar_t tgt_key = key[src_key_idx];
190
+ scalar_t tgt_value = value[src_value_idx];
191
+ if constexpr (is_fp8_e5m2_kv_cache) {
192
+#ifdef ENABLE_FP8_E5M2
193
+ key_cache[tgt_key_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_key);
194
+ value_cache[tgt_value_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_value);
195
+#else
196
+ assert(false);
197
+#endif
198
+ } else {
199
+ key_cache[tgt_key_idx] = tgt_key;
200
+ value_cache[tgt_value_idx] = tgt_value;
201
+ }
202
}
203
}
204
205
} // namespace vllm
206
207
+#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
208
+ vllm::reshape_and_cache_kernel<KV_T, CACHE_T, IS_FP8_E5M2_KV_CACHE><<<grid, block, 0, stream>>>( \
209
+ reinterpret_cast<KV_T*>(key.data_ptr()), \
210
+ reinterpret_cast<KV_T*>(value.data_ptr()), \
211
+ reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
212
+ reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
213
+ slot_mapping.data_ptr<int64_t>(), \
214
+ key_stride, \
215
+ value_stride, \
216
+ num_heads, \
217
+ head_size, \
218
+ block_size, \
219
+ x);
220
+
221
void reshape_and_cache(
222
torch::Tensor& key, // [num_tokens, num_heads, head_size]
223
torch::Tensor& value, // [num_tokens, num_heads, head_size]
224
torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
225
torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
200
- torch::Tensor& slot_mapping) // [num_tokens]
226
+ torch::Tensor& slot_mapping, // [num_tokens]
227
+ const std::string& kv_cache_dtype)
228
{
229
int num_tokens = key.size(0);
230
int num_heads = key.size(1);
@@ -212,23 +239,25 @@ void reshape_and_cache(
239
dim3 block(std::min(num_heads * head_size, 512));
240
const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
241
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
215
- VLLM_DISPATCH_FLOATING_TYPES(
216
- key.scalar_type(),
217
- "reshape_and_cache_kernel",
218
- [&] {
219
- vllm::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
220
- key.data_ptr<scalar_t>(),
221
- value.data_ptr<scalar_t>(),
222
- key_cache.data_ptr<scalar_t>(),
223
- value_cache.data_ptr<scalar_t>(),
224
- slot_mapping.data_ptr<int64_t>(),
225
- key_stride,
226
- value_stride,
227
- num_heads,
228
- head_size,
229
- block_size,
230
- x);
231
- });
242
+ if (kv_cache_dtype == "auto") {
243
+ if (key.dtype() == at::ScalarType::Float) {
244
+ CALL_RESHAPE_AND_CACHE(float, float, false);
245
+ } else if (key.dtype() == at::ScalarType::Half) {
246
+ CALL_RESHAPE_AND_CACHE(uint16_t, uint16_t, false);
247
+ } else if (key.dtype() == at::ScalarType::BFloat16) {
248
+ CALL_RESHAPE_AND_CACHE(__nv_bfloat16, __nv_bfloat16, false);
249
+ }
250
+ } else if (kv_cache_dtype == "fp8_e5m2") {
251
+ if (key.dtype() == at::ScalarType::Float) {
252
+ CALL_RESHAPE_AND_CACHE(float, uint8_t, true);
253
+ } else if (key.dtype() == at::ScalarType::Half) {
254
+ CALL_RESHAPE_AND_CACHE(uint16_t, uint8_t, true);
255
+ } else if (key.dtype() == at::ScalarType::BFloat16) {
256
+ CALL_RESHAPE_AND_CACHE(__nv_bfloat16, uint8_t, true);
257
+ }
258
+ } else {
259
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
260
+ }
261
}
262
263
namespace vllm {
@@ -256,12 +285,12 @@ __global__ void gather_cached_kv_kernel(
285
for (int i = threadIdx.x; i < num_tokens; i += blockDim.x) {
286
const int tgt_key_idx = token_idx * key_stride + i;
287
const int tgt_value_idx = token_idx * value_stride + i;
259
-
288
+
289
const int head_idx = i / head_size;
290
const int head_offset = i % head_size;
291
const int x_idx = head_offset / x; // the offset of the [head_size/x] dimension
292
const int x_offset = head_offset % x;
264
-
293
+
294
const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
295
+ head_idx * (head_size / x) * block_size * x
296
+ x_idx * block_size * x
@@ -373,7 +402,7 @@ void gather_cached_kv(
402
dim3 block(std::min(num_heads * head_size, 512));
403
const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
404
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
376
- VLLM_DISPATCH_FLOATING_TYPES(
405
+ VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
406
key.scalar_type(),
407
"gather_cached_kv_kernel_optimized",
408
[&] {
@@ -391,3 +420,55 @@ void gather_cached_kv(
420
x);
421
});
422
}
423
+
424
+namespace vllm {
425
+
426
+template<typename Tout, typename Tin>
427
+__global__ void convert_fp8_e5m2_kernel(
428
+ const Tin* __restrict__ src_cache,
429
+ Tout* __restrict__ dst_cache,
430
+ const int64_t block_stride) {
431
+ const int64_t block_idx = blockIdx.x;
432
+ for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
433
+ int64_t idx = block_idx * block_stride + i;
434
+#ifdef ENABLE_FP8_E5M2
435
+ dst_cache[idx] = fp8_e5m2_unscaled::vec_conversion<Tout, Tin>(src_cache[idx]);
436
+#else
437
+ assert(false);
438
+#endif
439
+ }
440
+}
441
+
442
+} // namespace vllm
443
+
444
+#define CALL_CONVERT_FP8_E5M2(Tout, Tin) \
445
+ vllm::convert_fp8_e5m2_kernel<Tout, Tin><<<grid, block, 0, stream>>>( \
446
+ reinterpret_cast<Tin*>(src_cache.data_ptr()), \
447
+ reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
448
+ block_stride);
449
+
450
+void convert_fp8_e5m2(
451
+ torch::Tensor& src_cache,
452
+ torch::Tensor& dst_cache)
453
+{
454
+ int64_t num_blocks = src_cache.size(0);
455
+ int64_t block_stride = src_cache.stride(0);
456
+
457
+ dim3 grid(num_blocks);
458
+ dim3 block(std::min(block_stride, int64_t(512)));
459
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
460
+
461
+ if (src_cache.dtype() == at::ScalarType::Float) {
462
+ CALL_CONVERT_FP8_E5M2(uint8_t, float);
463
+ } else if (src_cache.dtype() == at::ScalarType::Half) {
464
+ CALL_CONVERT_FP8_E5M2(uint8_t, uint16_t);
465
+ } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
466
+ CALL_CONVERT_FP8_E5M2(uint8_t, __nv_bfloat16);
467
+ } else if (dst_cache.dtype() == at::ScalarType::Float) {
468
+ CALL_CONVERT_FP8_E5M2(float, uint8_t);
469
+ } else if (dst_cache.dtype() == at::ScalarType::Half) {
470
+ CALL_CONVERT_FP8_E5M2(uint16_t, uint8_t);
471
+ } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
472
+ CALL_CONVERT_FP8_E5M2(__nv_bfloat16, uint8_t);
473
+ }
474
+}
csrc/dispatch_utils.h
+10
@@ -14,3 +14,13 @@
14
#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15
AT_DISPATCH_SWITCH( \
16
TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
17
+
18
+#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
19
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
20
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
21
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
22
+ AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__)
23
+
24
+#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
25
+ AT_DISPATCH_SWITCH( \
26
+ TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
csrc/ops.h
+4
-2
@@ -13,7 +13,8 @@ void paged_attention_v1(
13
torch::Tensor& context_lens,
14
int block_size,
15
int max_context_len,
16
- const c10::optional<torch::Tensor>& alibi_slopes);
16
+ const c10::optional<torch::Tensor>& alibi_slopes,
17
+ const std::string& kv_cache_dtype);
18
19
void paged_attention_v2(
20
torch::Tensor& out,
@@ -29,7 +30,8 @@ void paged_attention_v2(
30
torch::Tensor& context_lens,
31
int block_size,
32
int max_context_len,
32
- const c10::optional<torch::Tensor>& alibi_slopes);
33
+ const c10::optional<torch::Tensor>& alibi_slopes,
34
+ const std::string& kv_cache_dtype);
35
36
void rms_norm(
37
torch::Tensor& out,
csrc/pybind.cpp
+4
@@ -75,6 +75,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
75
"gather_cached_kv",
76
&gather_cached_kv,
77
"Gather key and value from the cache into contiguous QKV tensors");
78
+ cache_ops.def(
79
+ "convert_fp8_e5m2",
80
+ &convert_fp8_e5m2,
81
+ "Convert the key and value cache to fp8_e5m2 data type");
82
83
// Cuda utils
84
pybind11::module cuda_utils = m.def_submodule("cuda_utils", "vLLM cuda utils");
csrc/quantization/fp8_e5m2_kvcache/quant_utils.cuh
new
+278
@@ -0,0 +1,278 @@
1
+#pragma once
2
+
3
+#include <assert.h>
4
+#include <stdint.h>
5
+#include <float.h>
6
+#include <type_traits>
7
+#include "../../attention/attention_dtypes.h"
8
+#include "../../attention/dtype_float32.cuh"
9
+#include "../../attention/dtype_float16.cuh"
10
+#include "../../attention/dtype_bfloat16.cuh"
11
+
12
+#pragma once
13
+
14
+namespace vllm {
15
+#ifdef ENABLE_FP8_E5M2
16
+namespace fp8_e5m2_unscaled {
17
+
18
+template<typename Tout, typename Tin>
19
+__inline__ __device__ Tout vec_conversion(const Tin& x)
20
+{
21
+ return x;
22
+}
23
+
24
+// fp8 -> half
25
+template<>
26
+__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(const uint8_t& a)
27
+{
28
+ __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
29
+ return res.x;
30
+}
31
+
32
+// fp8x2 -> half2
33
+template<>
34
+__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(const uint16_t& a)
35
+{
36
+ union {
37
+ uint16_t u16[2];
38
+ uint32_t u32;
39
+ } tmp;
40
+ __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, __NV_E5M2);
41
+ tmp.u16[0] = res.x;
42
+ tmp.u16[1] = res.y;
43
+ return tmp.u32;
44
+}
45
+
46
+// fp8x4 -> half2x2
47
+template<>
48
+__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a)
49
+{
50
+ union {
51
+ uint2 u32x2;
52
+ uint32_t u32[2];
53
+ } tmp;
54
+ tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
55
+ tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
56
+ return tmp.u32x2;
57
+}
58
+
59
+// fp8x8 -> half2x4
60
+template<>
61
+__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a)
62
+{
63
+ union {
64
+ uint4 u64x2;
65
+ uint2 u64[2];
66
+ } tmp;
67
+ tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
68
+ tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
69
+ return tmp.u64x2;
70
+}
71
+
72
+// fp8 -> __nv_bfloat16
73
+template<>
74
+__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a)
75
+{
76
+ // Note there is no direct convert function from fp8 to bf16.
77
+ // fp8 -> half
78
+ __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
79
+ // half -> float -> bf16
80
+ float tmp = half_to_float(res.x);
81
+ return __float2bfloat16(tmp);
82
+}
83
+
84
+// fp8x2 -> __nv_bfloat162
85
+template<>
86
+__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a)
87
+{
88
+ __nv_bfloat162 res;
89
+ res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
90
+ res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
91
+ return res;
92
+}
93
+
94
+// fp8x4 -> bf16_4_t
95
+template<>
96
+__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a)
97
+{
98
+ bf16_4_t res;
99
+ res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
100
+ res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
101
+ return res;
102
+}
103
+
104
+// fp8x8 -> bf16_8_t
105
+template<>
106
+__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a)
107
+{
108
+ bf16_4_t tmp1, tmp2;
109
+ tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
110
+ tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
111
+ bf16_8_t res;
112
+ res.x = tmp1.x;
113
+ res.y = tmp1.y;
114
+ res.z = tmp2.x;
115
+ res.w = tmp2.y;
116
+ return res;
117
+}
118
+
119
+// fp8 -> float
120
+template<>
121
+__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a)
122
+{
123
+ // fp8 -> half
124
+ uint16_t tmp = vec_conversion<uint16_t, uint8_t>(a);
125
+ // half -> float
126
+ return half_to_float(tmp);
127
+}
128
+
129
+// fp8x2 -> float2
130
+template<>
131
+__inline__ __device__ float2 vec_conversion<float2, uint16_t>(const uint16_t& a)
132
+{
133
+ // fp8x2 -> half2
134
+ uint32_t tmp = vec_conversion<uint32_t, uint16_t>(a);
135
+ // half2 -> float2
136
+ return half2_to_float2(tmp);
137
+}
138
+
139
+// fp8x4 -> float4
140
+template<>
141
+__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(const uint32_t& a)
142
+{
143
+ Float4_ res;
144
+ res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
145
+ res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
146
+ return res;
147
+}
148
+
149
+// fp8x8 -> float8
150
+template<>
151
+__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a)
152
+{
153
+ Float4_ tmp1, tmp2;
154
+ tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
155
+ tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
156
+ Float8_ res;
157
+ res.x = tmp1.x;
158
+ res.y = tmp1.y;
159
+ res.z = tmp2.x;
160
+ res.w = tmp2.y;
161
+ return res;
162
+}
163
+
164
+
165
+// half -> fp8
166
+template<>
167
+__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(const uint16_t& a)
168
+{
169
+ __half_raw tmp;
170
+ tmp.x = a;
171
+ __nv_fp8_storage_t res = __nv_cvt_halfraw_to_fp8(tmp, __NV_SATFINITE, __NV_E5M2);
172
+ return (uint8_t)res;
173
+}
174
+
175
+// bf16 -> fp8
176
+template<>
177
+__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a)
178
+{
179
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
180
+ assert(false);
181
+#else
182
+ __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(__nv_bfloat16_raw(a), __NV_SATFINITE, __NV_E5M2);
183
+ return (uint8_t)res;
184
+#endif
185
+}
186
+
187
+// float -> fp8
188
+template<>
189
+__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a)
190
+{
191
+ __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(a, __NV_SATFINITE, __NV_E5M2);
192
+ return (uint8_t)res;
193
+}
194
+
195
+// fp8x4 -> float4
196
+template<>
197
+__inline__ __device__ float4 vec_conversion<float4, uint32_t>(const uint32_t& a)
198
+{
199
+ Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
200
+ float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
201
+ return res;
202
+}
203
+
204
+
205
+template<>
206
+__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(const float2& a)
207
+{
208
+ union {
209
+ half2 float16;
210
+ uint32_t uint32;
211
+ };
212
+
213
+ float16 = __float22half2_rn(a);
214
+ return uint32;
215
+}
216
+
217
+template<>
218
+__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a)
219
+{
220
+ uint2 b;
221
+ float2 val;
222
+ val.x = a.x.x;
223
+ val.y = a.x.y;
224
+ b.x = vec_conversion<uint32_t, float2>(val);
225
+
226
+ val.x = a.y.x;
227
+ val.y = a.y.y;
228
+ b.y = vec_conversion<uint32_t, float2>(val);
229
+
230
+ return b;
231
+}
232
+
233
+template<>
234
+__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a)
235
+{
236
+ float4 b;
237
+ b.x = a.x.x;
238
+ b.y = a.x.y;
239
+ b.z = a.y.x;
240
+ b.w = a.y.y;
241
+ return b;
242
+}
243
+
244
+template<>
245
+__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a)
246
+{
247
+ uint4 b;
248
+ b.x = vec_conversion<uint32_t, float2>(a.x);
249
+ b.y = vec_conversion<uint32_t, float2>(a.y);
250
+ b.z = vec_conversion<uint32_t, float2>(a.z);
251
+ b.w = vec_conversion<uint32_t, float2>(a.w);
252
+ return b;
253
+}
254
+
255
+template<>
256
+__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(const float2 &a) {
257
+ __nv_bfloat162 b;
258
+ from_float(b, a);
259
+ return b;
260
+}
261
+
262
+template<>
263
+__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(const Float4_ &a) {
264
+ bf16_4_t b;
265
+ from_float(b, a);
266
+ return b;
267
+}
268
+
269
+template<>
270
+__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(const Float8_ &a) {
271
+ bf16_8_t b;
272
+ from_float(b, a);
273
+ return b;
274
+}
275
+
276
+} // namespace fp8_e5m2_unscaled
277
+#endif // ENABLE_FP8_E5M2
278
+} // namespace vllm
docs/source/quantization/fp8_e5m2_kv_cache.rst
new
+32
@@ -0,0 +1,32 @@
1
+.. _fp8_e5m2_kv_cache:
2
+
3
+FP8 E5M2 KV Cache
4
+==================
5
+
6
+The int8/int4 quantization scheme requires additional scale GPU memory storage, which reduces the expected GPU memory benefits.
7
+The FP8 data format retains 2~3 mantissa bits and can convert float/fp16/bflaot16 and fp8 to each other.
8
+
9
+Here is an example of how to enable this feature:
10
+
11
+.. code-block:: python
12
+ from vllm import LLM, SamplingParams
13
+ # Sample prompts.
14
+ prompts = [
15
+ "Hello, my name is",
16
+ "The president of the United States is",
17
+ "The capital of France is",
18
+ "The future of AI is",
19
+ ]
20
+ # Create a sampling params object.
21
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22
+ # Create an LLM.
23
+ llm = LLM(model="facebook/opt-125m", kv_cache_dtype="fp8_e5m2")
24
+ # Generate texts from the prompts. The output is a list of RequestOutput objects
25
+ # that contain the prompt, generated text, and other information.
26
+ outputs = llm.generate(prompts, sampling_params)
27
+ # Print the outputs.
28
+ for output in outputs:
29
+ prompt = output.prompt
30
+ generated_text = output.outputs[0].text
31
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
32
+
setup.py
+3
@@ -253,6 +253,9 @@ if _is_cuda():
253
num_threads = min(os.cpu_count(), nvcc_threads)
254
NVCC_FLAGS += ["--threads", str(num_threads)]
255
256
+ if nvcc_cuda_version >= Version("11.8"):
257
+ NVCC_FLAGS += ["-DENABLE_FP8_E5M2"]
258
+
259
# changes for punica kernels
260
NVCC_FLAGS += torch_cpp_ext.COMMON_NVCC_FLAGS
261
REMOVE_NVCC_FLAGS = [
tests/kernels/conftest.py
+2
-39
@@ -1,44 +1,7 @@
1
-from typing import List, Tuple
2
-
1
import pytest
4
-import torch
5
-
6
-
7
-def create_kv_caches(
8
- num_blocks: int,
9
- block_size: int,
10
- num_layers: int,
11
- num_heads: int,
12
- head_size: int,
13
- dtype: torch.dtype,
14
- seed: int,
15
- device: str,
16
-) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
17
- torch.random.manual_seed(seed)
18
- torch.cuda.manual_seed(seed)
19
-
20
- scale = head_size**-0.5
21
- x = 16 // torch.tensor([], dtype=dtype).element_size()
22
- key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
23
- key_caches = []
24
- for _ in range(num_layers):
25
- key_cache = torch.empty(size=key_cache_shape,
26
- dtype=dtype,
27
- device=device)
28
- key_cache.uniform_(-scale, scale)
29
- key_caches.append(key_cache)
30
-
31
- value_cache_shape = (num_blocks, num_heads, head_size, block_size)
32
- value_caches = []
33
- for _ in range(num_layers):
34
- value_cache = torch.empty(size=value_cache_shape,
35
- dtype=dtype,
36
- device=device)
37
- value_cache.uniform_(-scale, scale)
38
- value_caches.append(value_cache)
39
- return key_caches, value_caches
2
+from vllm.utils import create_kv_caches_with_random
3
4
5
@pytest.fixture()
6
def kv_cache_factory():
44
- return create_kv_caches
7
+ return create_kv_caches_with_random
tests/kernels/test_attention.py
+36
-5
@@ -6,14 +6,16 @@ import torch
6
from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9
-from vllm._C import ops
9
+from vllm._C import ops, cache_ops
10
from vllm.utils import get_max_shared_memory_bytes
11
12
FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
13
# This will change depending on the compute capability.
14
# - 512 as a buffer
15
MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
16
-NUM_BLOCKS = 12000 # Arbitrary values for testing
16
+# There may not be enough gpu memory due to large NUM_BLOCKS.
17
+# Reduce NUM_BLOCKS when it happens.
18
+NUM_BLOCKS = 4321 # Arbitrary values for testing
19
PARTITION_SIZE = 512
20
21
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -23,6 +25,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
25
HEAD_SIZES = [64, 80, 96, 112, 128, 256]
26
BLOCK_SIZES = [16, 32]
27
USE_ALIBI = [False, True]
28
+KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
29
SEEDS = [0]
30
DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
31
@@ -105,6 +108,7 @@ def ref_single_query_cached_kv_attention(
108
@pytest.mark.parametrize("use_alibi", USE_ALIBI)
109
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
110
@pytest.mark.parametrize("dtype", DTYPES)
111
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
112
@pytest.mark.parametrize("seed", SEEDS)
113
@pytest.mark.parametrize("device", DEVICES)
114
def test_paged_attention(
@@ -116,6 +120,7 @@ def test_paged_attention(
120
use_alibi: bool,
121
block_size: int,
122
dtype: torch.dtype,
123
+ kv_cache_dtype: str,
124
seed: int,
125
device: int,
126
) -> None:
@@ -158,8 +163,9 @@ def test_paged_attention(
163
164
# Create the KV caches.
165
key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
161
- num_kv_heads, head_size, dtype,
162
- seed, gpu_id)
166
+ num_kv_heads, head_size,
167
+ kv_cache_dtype, dtype, seed,
168
+ gpu_id)
169
key_cache, value_cache = key_caches[0], value_caches[0]
170
171
# Call the paged attention kernel.
@@ -177,6 +183,7 @@ def test_paged_attention(
183
block_size,
184
max_context_len,
185
alibi_slopes,
186
+ kv_cache_dtype,
187
)
188
elif version == "v2":
189
num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
@@ -209,11 +216,30 @@ def test_paged_attention(
216
block_size,
217
max_context_len,
218
alibi_slopes,
219
+ kv_cache_dtype,
220
)
221
else:
222
raise AssertionError(f"Unknown version: {version}")
223
224
# Run the reference implementation.
225
+ if kv_cache_dtype == "fp8_e5m2":
226
+ # Convert cache data back to dtype.
227
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
228
+ key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
229
+ block_size, x)
230
+ dequantized_key_cache = torch.empty(size=key_cache_shape,
231
+ dtype=dtype,
232
+ device=gpu_id)
233
+ cache_ops.convert_fp8_e5m2(key_cache, dequantized_key_cache)
234
+ key_cache = dequantized_key_cache
235
+
236
+ value_cache_shape = value_cache.shape
237
+ dequantized_value_cache = torch.empty(size=value_cache_shape,
238
+ dtype=dtype,
239
+ device=gpu_id)
240
+ cache_ops.convert_fp8_e5m2(value_cache, dequantized_value_cache)
241
+ value_cache = dequantized_value_cache
242
+
243
ref_output = torch.empty_like(query)
244
ref_single_query_cached_kv_attention(
245
ref_output,
@@ -230,7 +256,12 @@ def test_paged_attention(
256
# NOTE(woosuk): Due to the kernel-level differences in the two
257
# implementations, there is a small numerical difference in the two
258
# outputs. Thus, we use a relaxed tolerance for the test.
233
- assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
259
+ # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
260
+ # so we use a relaxed tolerance for the test.
261
+ atol, rtol = 1e-3, 1e-5
262
+ if kv_cache_dtype == "fp8_e5m2":
263
+ atol, rtol = 1e-2, 1e-5
264
+ assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
265
266
267
def ref_multi_query_kv_attention(
tests/kernels/test_cache.py
+7
-3
@@ -15,6 +15,7 @@ NUM_BLOCKS = [1024, 3600] # Arbitrary values for testing
15
NUM_MAPPINGS = [256] # Arbitrary values for testing
16
SEEDS = [0]
17
DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
18
+KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
19
20
21
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@@ -26,6 +27,7 @@ DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
27
@pytest.mark.parametrize("dtype", DTYPES)
28
@pytest.mark.parametrize("seed", SEEDS)
29
@pytest.mark.parametrize("device", DEVICES)
30
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
31
@torch.inference_mode()
32
def test_copy_blocks(
33
kv_cache_factory,
@@ -38,6 +40,7 @@ def test_copy_blocks(
40
dtype: torch.dtype,
41
seed: int,
42
device: int,
43
+ kv_cache_dtype: str,
44
) -> None:
45
random.seed(seed)
46
torch.random.manual_seed(seed)
@@ -59,7 +62,8 @@ def test_copy_blocks(
62
# Create the KV caches.
63
key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
64
num_layers, num_heads,
62
- head_size, dtype, seed, gpu_id)
65
+ head_size, kv_cache_dtype,
66
+ dtype, seed, gpu_id)
67
68
# Clone the KV caches.
69
cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
@@ -124,7 +128,7 @@ def test_reshape_and_cache(
128
# Create the KV caches.
129
key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
130
num_heads, head_size, dtype,
127
- seed, gpu_id)
131
+ None, seed, gpu_id)
132
key_cache, value_cache = key_caches[0], value_caches[0]
133
134
# Clone the KV caches.
@@ -133,7 +137,7 @@ def test_reshape_and_cache(
137
138
# Call the reshape_and_cache kernel.
139
cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
136
- slot_mapping)
140
+ slot_mapping, "auto")
141
142
# Run the reference implementation.
143
reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
vllm/config.py
+28
-1
@@ -1,13 +1,14 @@
1
from typing import Optional, Union, ClassVar
2
from dataclasses import dataclass
3
import os
4
+from packaging.version import Version
5
6
import torch
7
from transformers import PretrainedConfig
8
9
from vllm.logger import init_logger
10
from vllm.transformers_utils.config import get_config
10
-from vllm.utils import get_cpu_memory, is_hip
11
+from vllm.utils import get_cpu_memory, is_hip, get_nvcc_cuda_version
12
13
logger = init_logger(__name__)
14
@@ -275,6 +276,7 @@ class CacheConfig:
276
gpu_memory_utilization: Fraction of GPU memory to use for the
277
vLLM execution.
278
swap_space: Size of the CPU swap space per GPU (in GiB).
279
+ cache_dtype: Data type for kv cache storage.
280
"""
281
282
def __init__(
@@ -282,13 +284,16 @@ class CacheConfig:
284
block_size: int,
285
gpu_memory_utilization: float,
286
swap_space: int,
287
+ cache_dtype: str,
288
sliding_window: Optional[int] = None,
289
) -> None:
290
self.block_size = block_size
291
self.gpu_memory_utilization = gpu_memory_utilization
292
self.swap_space_bytes = swap_space * _GB
293
+ self.cache_dtype = cache_dtype
294
self.sliding_window = sliding_window
295
self._verify_args()
296
+ self._verify_cache_dtype()
297
298
# Will be set after profiling.
299
self.num_gpu_blocks = None
@@ -300,6 +305,28 @@ class CacheConfig:
305
"GPU memory utilization must be less than 1.0. Got "
306
f"{self.gpu_memory_utilization}.")
307
308
+ def _verify_cache_dtype(self) -> None:
309
+ if self.cache_dtype == "auto":
310
+ pass
311
+ elif self.cache_dtype == "fp8_e5m2":
312
+ nvcc_cuda_version = get_nvcc_cuda_version()
313
+ if nvcc_cuda_version < Version("11.8"):
314
+ raise ValueError(
315
+ "FP8 is not supported when cuda version is lower than 11.8."
316
+ )
317
+ device_name = torch.cuda.get_device_name()
318
+ if "AMD" in device_name:
319
+ raise NotImplementedError(
320
+ "FP8_E5M2 KV Cache on AMD GPU has not been supported yet.")
321
+ logger.info(
322
+ "Using fp8_e5m2 data type to store kv cache. It reduces "
323
+ "the GPU memory footprint and boosts the performance. "
324
+ "But it may cause slight accuracy drop. "
325
+ "Currently we only support fp8 without scaling factors and "
326
+ "make e5m2 as a default format.")
327
+ else:
328
+ raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}")
329
+
330
def verify_with_parallel_config(
331
self,
332
parallel_config: "ParallelConfig",
vllm/engine/arg_utils.py
+10
-1
@@ -17,6 +17,7 @@ class EngineArgs:
17
download_dir: Optional[str] = None
18
load_format: str = 'auto'
19
dtype: str = 'auto'
20
+ kv_cache_dtype: str = 'auto'
21
seed: int = 0
22
max_model_len: Optional[int] = None
23
worker_use_ray: bool = False
@@ -122,6 +123,14 @@ class EngineArgs:
123
'The "auto" option will use FP16 precision '
124
'for FP32 and FP16 models, and BF16 precision '
125
'for BF16 models.')
126
+ parser.add_argument(
127
+ '--kv-cache-dtype',
128
+ type=str,
129
+ choices=['auto', 'fp8_e5m2'],
130
+ default='auto',
131
+ help='Data type for kv cache storage. If "auto", will use model '
132
+ 'data type. Note FP8 is not supported when cuda version is '
133
+ 'lower than 11.8.')
134
parser.add_argument('--max-model-len',
135
type=int,
136
default=None,
@@ -269,7 +278,7 @@ class EngineArgs:
278
self.max_context_len_to_capture)
279
cache_config = CacheConfig(self.block_size,
280
self.gpu_memory_utilization,
272
- self.swap_space,
281
+ self.swap_space, self.kv_cache_dtype,
282
model_config.get_sliding_window())
283
parallel_config = ParallelConfig(self.pipeline_parallel_size,
284
self.tensor_parallel_size,
vllm/engine/llm_engine.py
+6
@@ -85,6 +85,7 @@ class LLMEngine:
85
f"disable_custom_all_reduce={parallel_config.disable_custom_all_reduce}, "
86
f"quantization={model_config.quantization}, "
87
f"enforce_eager={model_config.enforce_eager}, "
88
+ f"kv_cache_dtype={cache_config.cache_dtype}, "
89
f"seed={model_config.seed})")
90
# TODO(woosuk): Print more configs in debug mode.
91
@@ -144,6 +145,7 @@ class LLMEngine:
145
rank=0,
146
distributed_init_method=distributed_init_method,
147
lora_config=self.lora_config,
148
+ kv_cache_dtype=self.cache_config.cache_dtype,
149
is_driver_worker=True,
150
)
151
self._run_workers("init_model")
@@ -234,6 +236,7 @@ class LLMEngine:
236
model_config = copy.deepcopy(self.model_config)
237
parallel_config = copy.deepcopy(self.parallel_config)
238
scheduler_config = copy.deepcopy(self.scheduler_config)
239
+ cache_config = copy.deepcopy(self.cache_config)
240
241
for rank, (worker, (node_id,
242
_)) in enumerate(zip(self.workers,
@@ -249,6 +252,7 @@ class LLMEngine:
252
rank,
253
distributed_init_method,
254
lora_config=self.lora_config,
255
+ cache_config=cache_config,
256
))
257
258
driver_rank = 0
@@ -261,6 +265,7 @@ class LLMEngine:
265
driver_rank,
266
distributed_init_method,
267
lora_config=self.lora_config,
268
+ cache_config=cache_config,
269
is_driver_worker=True,
270
)
271
@@ -306,6 +311,7 @@ class LLMEngine:
311
block_size=self.cache_config.block_size,
312
gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
313
cpu_swap_space=self.cache_config.swap_space_bytes,
314
+ cache_dtype=self.cache_config.cache_dtype,
315
)
316
317
# Since we use a shared centralized controller, we take the minimum
vllm/model_executor/input_metadata.py
+5
-1
@@ -12,6 +12,7 @@ class InputMetadata:
12
max_context_len: The maximum context length.
13
context_lens: the length of attention context for each sequence.
14
block_tables: The block tables. (Seq id -> list of physical block)
15
+ kv_cache_dtype: Data type to store kv cache.
16
"""
17
18
def __init__(
@@ -25,6 +26,7 @@ class InputMetadata:
26
context_lens: Optional[torch.Tensor],
27
block_tables: Optional[torch.Tensor],
28
use_cuda_graph: bool,
29
+ kv_cache_dtype: str,
30
) -> None:
31
self.is_prompt = is_prompt
32
self.prompt_lens = prompt_lens
@@ -35,6 +37,7 @@ class InputMetadata:
37
self.context_lens = context_lens
38
self.block_tables = block_tables
39
self.use_cuda_graph = use_cuda_graph
40
+ self.kv_cache_dtype = kv_cache_dtype
41
42
# Set during the execution of the first attention op.
43
# FIXME(woosuk): This is a hack.
@@ -47,4 +50,5 @@ class InputMetadata:
50
f"slot_mapping={self.slot_mapping}, "
51
f"context_lens={self.context_lens}, "
52
f"block_tables={self.block_tables}, "
50
- f"use_cuda_graph={self.use_cuda_graph})")
53
+ f"use_cuda_graph={self.use_cuda_graph}, "
54
+ f"kv_cache_dtype={self.kv_cache_dtype})")
vllm/model_executor/layers/attention.py
+3
@@ -98,6 +98,7 @@ class PagedAttention(nn.Module):
98
key_cache,
99
value_cache,
100
input_metadata.slot_mapping.flatten(),
101
+ input_metadata.kv_cache_dtype,
102
)
103
104
if input_metadata.is_prompt:
@@ -265,6 +266,7 @@ def _paged_attention(
266
block_size,
267
input_metadata.max_context_len,
268
alibi_slopes,
269
+ input_metadata.kv_cache_dtype,
270
)
271
else:
272
# Run PagedAttention V2.
@@ -295,5 +297,6 @@ def _paged_attention(
297
block_size,
298
input_metadata.max_context_len,
299
alibi_slopes,
300
+ input_metadata.kv_cache_dtype,
301
)
302
return output
vllm/utils.py
+109
-1
@@ -1,9 +1,11 @@
1
import enum
2
import os
3
import socket
4
+import subprocess
5
import uuid
6
from platform import uname
6
-from typing import List
7
+from typing import List, Tuple, Union
8
+from packaging.version import parse, Version
9
10
import psutil
11
import torch
@@ -17,7 +19,17 @@ from typing import (
19
from collections import OrderedDict
20
from typing import Any, Hashable, Optional
21
22
+from vllm.logger import init_logger
23
+
24
T = TypeVar("T")
25
+logger = init_logger(__name__)
26
+
27
+STR_DTYPE_TO_TORCH_DTYPE = {
28
+ "half": torch.half,
29
+ "bfloat16": torch.bfloat16,
30
+ "float": torch.float,
31
+ "fp8_e5m2": torch.uint8,
32
+}
33
34
35
class Device(enum.Enum):
@@ -167,3 +179,99 @@ def get_open_port() -> int:
179
180
def set_cuda_visible_devices(device_ids: List[int]) -> None:
181
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
182
+
183
+
184
+def get_nvcc_cuda_version() -> Version:
185
+ cuda_home = os.environ.get('CUDA_HOME')
186
+ if not cuda_home:
187
+ cuda_home = '/usr/local/cuda'
188
+ logger.info(
189
+ f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
190
+ )
191
+ nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
192
+ universal_newlines=True)
193
+ output = nvcc_output.split()
194
+ release_idx = output.index("release") + 1
195
+ nvcc_cuda_version = parse(output[release_idx].split(",")[0])
196
+ return nvcc_cuda_version
197
+
198
+
199
+def _generate_random_fp8_e5m2(
200
+ tensor: torch.tensor,
201
+ low: float,
202
+ high: float,
203
+) -> None:
204
+ # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,
205
+ # it may occur Inf or NaN if we directly use torch.randint
206
+ # to generate random data for fp8 data.
207
+ # For example, s.11111.00 in fp8e5m2 format repesents Inf.
208
+ # | E4M3 | E5M2
209
+ #-----|-------------|-------------------
210
+ # Inf | N/A | s.11111.00
211
+ # NaN | s.1111.111 | s.11111.{01,10,11}
212
+ from vllm._C import cache_ops
213
+ tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
214
+ tensor_tmp.uniform_(low, high)
215
+ cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)
216
+ del tensor_tmp
217
+
218
+
219
+def create_kv_caches_with_random(
220
+ num_blocks: int,
221
+ block_size: int,
222
+ num_layers: int,
223
+ num_heads: int,
224
+ head_size: int,
225
+ cache_dtype: Optional[Union[str, torch.dtype]],
226
+ model_dtype: Optional[Union[str, torch.dtype]] = None,
227
+ seed: Optional[int] = 0,
228
+ device: Optional[str] = "cuda",
229
+) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
230
+ torch.random.manual_seed(seed)
231
+ torch.cuda.manual_seed(seed)
232
+
233
+ if isinstance(cache_dtype, str):
234
+ if cache_dtype == "auto":
235
+ if isinstance(model_dtype, str):
236
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]
237
+ elif isinstance(model_dtype, torch.dtype):
238
+ torch_dtype = model_dtype
239
+ else:
240
+ raise ValueError(f"Invalid model dtype: {model_dtype}")
241
+ elif cache_dtype in ["half", "bfloat16", "float"]:
242
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
243
+ elif cache_dtype == "fp8_e5m2":
244
+ torch_dtype = torch.uint8
245
+ else:
246
+ raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
247
+ elif isinstance(cache_dtype, torch.dtype):
248
+ torch_dtype = cache_dtype
249
+ else:
250
+ raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
251
+
252
+ scale = head_size**-0.5
253
+ x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
254
+ key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
255
+ key_caches = []
256
+ for _ in range(num_layers):
257
+ key_cache = torch.empty(size=key_cache_shape,
258
+ dtype=torch_dtype,
259
+ device=device)
260
+ if cache_dtype in ["auto", "half", "bfloat16", "float"]:
261
+ key_cache.uniform_(-scale, scale)
262
+ elif cache_dtype == 'fp8_e5m2':
263
+ _generate_random_fp8_e5m2(key_cache, -scale, scale)
264
+ key_caches.append(key_cache)
265
+
266
+ value_cache_shape = (num_blocks, num_heads, head_size, block_size)
267
+ value_caches = []
268
+ for _ in range(num_layers):
269
+ value_cache = torch.empty(size=value_cache_shape,
270
+ dtype=torch_dtype,
271
+ device=device)
272
+ if cache_dtype in ["auto", "half", "bfloat16", "float"]:
273
+ value_cache.uniform_(-scale, scale)
274
+ elif cache_dtype == 'fp8_e5m2':
275
+ _generate_random_fp8_e5m2(value_cache, -scale, scale)
276
+ value_caches.append(value_cache)
277
+ return key_caches, value_caches
vllm/worker/cache_engine.py
+12
-3
@@ -6,7 +6,7 @@ import torch
6
from vllm._C import cache_ops
7
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
from vllm.logger import init_logger
9
-from vllm.utils import in_wsl
9
+from vllm.utils import in_wsl, STR_DTYPE_TO_TORCH_DTYPE
10
11
logger = init_logger(__name__)
12
@@ -34,12 +34,16 @@ class CacheEngine:
34
self.head_size = model_config.get_head_size()
35
self.num_layers = model_config.get_num_layers(parallel_config)
36
self.num_heads = model_config.get_num_kv_heads(parallel_config)
37
- self.dtype = model_config.dtype
37
38
self.block_size = cache_config.block_size
39
self.num_gpu_blocks = cache_config.num_gpu_blocks
40
self.num_cpu_blocks = cache_config.num_cpu_blocks
41
42
+ if cache_config.cache_dtype == "auto":
43
+ self.dtype = model_config.dtype
44
+ else:
45
+ self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
46
+
47
# Initialize the cache.
48
self.gpu_cache = self.allocate_gpu_cache()
49
self.cpu_cache = self.allocate_cpu_cache()
@@ -142,6 +146,7 @@ class CacheEngine:
146
@staticmethod
147
def get_cache_block_size(
148
block_size: int,
149
+ cache_dtype: str,
150
model_config: ModelConfig,
151
parallel_config: ParallelConfig,
152
) -> int:
@@ -152,7 +157,11 @@ class CacheEngine:
157
key_cache_block = block_size * num_heads * head_size
158
value_cache_block = key_cache_block
159
total = num_layers * (key_cache_block + value_cache_block)
155
- dtype_size = _get_dtype_size(model_config.dtype)
160
+ if cache_dtype == "auto":
161
+ dtype = model_config.dtype
162
+ else:
163
+ dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
164
+ dtype_size = _get_dtype_size(dtype)
165
return dtype_size * total
166
167
vllm/worker/model_runner.py
+7
@@ -36,6 +36,7 @@ class ModelRunner:
36
parallel_config: ParallelConfig,
37
scheduler_config: SchedulerConfig,
38
lora_config: Optional[LoRAConfig],
39
+ kv_cache_dtype: Optional[str] = "auto",
40
is_driver_worker: bool = False,
41
):
42
self.model_config = model_config
@@ -68,6 +69,7 @@ class ModelRunner:
69
self.graph_block_tables = None # Set after initial profiling.
70
# cache in_wsl result
71
self.in_wsl = in_wsl()
72
+ self.kv_cache_dtype = kv_cache_dtype
73
74
def load_model(self) -> None:
75
self.model = get_model(self.model_config, self.lora_config)
@@ -223,6 +225,7 @@ class ModelRunner:
225
context_lens=context_lens_tensor,
226
block_tables=block_tables,
227
use_cuda_graph=False,
228
+ kv_cache_dtype=self.kv_cache_dtype,
229
)
230
return (input_tokens, input_positions, input_metadata, prompt_lens,
231
subquery_lens, lora_index_mapping, lora_prompt_mapping,
@@ -350,6 +353,7 @@ class ModelRunner:
353
context_lens=context_lens,
354
block_tables=block_tables,
355
use_cuda_graph=use_captured_graph,
356
+ kv_cache_dtype=self.kv_cache_dtype,
357
)
358
return input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, lora_requests
359
@@ -473,6 +477,7 @@ class ModelRunner:
477
"context_lens": input_metadata.context_lens,
478
"block_tables": input_metadata.block_tables,
479
"use_cuda_graph": input_metadata.use_cuda_graph,
480
+ "kv_cache_dtype": input_metadata.kv_cache_dtype,
481
"selected_token_indices":
482
sampling_metadata.selected_token_indices,
483
"lora_requests": lora_requests,
@@ -495,6 +500,7 @@ class ModelRunner:
500
context_lens=metadata_dict["context_lens"],
501
block_tables=metadata_dict["block_tables"],
502
use_cuda_graph=metadata_dict["use_cuda_graph"],
503
+ kv_cache_dtype=metadata_dict["kv_cache_dtype"],
504
)
505
sampling_metadata = SamplingMetadata(
506
seq_groups=None,
@@ -665,6 +671,7 @@ class ModelRunner:
671
context_lens=context_lens[:batch_size],
672
block_tables=block_tables[:batch_size],
673
use_cuda_graph=True,
674
+ kv_cache_dtype=self.kv_cache_dtype,
675
)
676
677
if self.lora_config:
vllm/worker/worker.py
+4
-1
@@ -37,6 +37,7 @@ class Worker:
37
rank: int,
38
distributed_init_method: str,
39
lora_config: Optional[LoRAConfig] = None,
40
+ kv_cache_dtype: Optional[str] = "auto",
41
is_driver_worker: bool = False,
42
) -> None:
43
self.model_config = model_config
@@ -54,6 +55,7 @@ class Worker:
55
parallel_config,
56
scheduler_config,
57
lora_config=self.lora_config,
58
+ kv_cache_dtype=kv_cache_dtype,
59
is_driver_worker=is_driver_worker)
60
# Uninitialized cache engine. Will be initialized by
61
# self.init_cache_engine().
@@ -95,6 +97,7 @@ class Worker:
97
block_size: int,
98
gpu_memory_utilization: float,
99
cpu_swap_space: int,
100
+ cache_dtype: str,
101
) -> Tuple[int, int]:
102
"""Profiles the peak memory usage of the model and returns the maximum
103
number of GPU and CPU cache blocks that can be allocated.
@@ -119,7 +122,7 @@ class Worker:
122
peak_memory = total_gpu_memory - free_gpu_memory
123
124
cache_block_size = CacheEngine.get_cache_block_size(
122
- block_size, self.model_config, self.parallel_config)
125
+ block_size, cache_dtype, self.model_config, self.parallel_config)
126
num_gpu_blocks = int(
127
(total_gpu_memory * gpu_memory_utilization - peak_memory) //
128
cache_block_size)