@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9090bf02e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Support FP8-E5M2 KV Cache (#2279)

Co-authored-by: zhaoyang <zhao.yang16@zte.com.cn> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

zhaoyang-star committed Jan 29, 2024 at 08:43 UTC 9090bf02e74334a8020b454814e0d00fa780fd79
26 files changed +912 -196
benchmarks/benchmark_latency.py
+8
@@ -24,6 +24,7 @@ def main(args: argparse.Namespace):
24 trust_remote_code=args.trust_remote_code,
25 dtype=args.dtype,
26 enforce_eager=args.enforce_eager,
27 + kv_cache_dtype=args.kv_cache_dtype,
28 )
29
30 sampling_params = SamplingParams(
@@ -117,6 +118,13 @@ if __name__ == '__main__':
118 parser.add_argument('--enforce-eager',
119 action='store_true',
120 help='enforce eager mode and disable CUDA graph')
121 + parser.add_argument(
122 + "--kv-cache-dtype",
123 + type=str,
124 + choices=['auto', 'fp8_e5m2'],
125 + default='auto',
126 + help=
127 + 'Data type for kv cache storage. If "auto", will use model data type.')
128 parser.add_argument(
129 '--profile',
130 action='store_true',
benchmarks/benchmark_throughput.py
+11 -1
@@ -71,6 +71,7 @@ def run_vllm(
71 dtype: str,
72 max_model_len: Optional[int],
73 enforce_eager: bool,
74 + kv_cache_dtype: str,
75 ) -> float:
76 from vllm import LLM, SamplingParams
77 llm = LLM(
@@ -83,6 +84,7 @@ def run_vllm(
84 dtype=dtype,
85 max_model_len=max_model_len,
86 enforce_eager=enforce_eager,
87 + kv_cache_dtype=kv_cache_dtype,
88 )
89
90 # Add the requests to the engine.
@@ -206,7 +208,8 @@ def main(args: argparse.Namespace):
208 args.quantization, args.tensor_parallel_size,
209 args.seed, args.n, args.use_beam_search,
210 args.trust_remote_code, args.dtype,
209 - args.max_model_len, args.enforce_eager)
211 + args.max_model_len, args.enforce_eager,
212 + args.kv_cache_dtype)
213 elif args.backend == "hf":
214 assert args.tensor_parallel_size == 1
215 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -284,6 +287,13 @@ if __name__ == "__main__":
287 parser.add_argument("--enforce-eager",
288 action="store_true",
289 help="enforce eager execution")
290 + parser.add_argument(
291 + "--kv-cache-dtype",
292 + type=str,
293 + choices=["auto", "fp8_e5m2"],
294 + default="auto",
295 + help=
296 + 'Data type for kv cache storage. If "auto", will use model data type.')
297 args = parser.parse_args()
298 if args.tokenizer is None:
299 args.tokenizer = args.model
benchmarks/kernels/benchmark_paged_attention.py
+18 -15
@@ -1,9 +1,11 @@
1 +from typing import Optional
2 import argparse
3 import random
4 import time
5
6 import torch
7
8 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
9 from vllm._C import ops
10
11 NUM_BLOCKS = 1024
@@ -23,6 +25,7 @@ def main(
25 dtype: torch.dtype,
26 seed: int,
27 do_profile: bool,
28 + kv_cache_dtype: Optional[str] = None,
29 ) -> None:
30 random.seed(seed)
31 torch.random.manual_seed(seed)
@@ -59,15 +62,10 @@ def main(
62 block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
63
64 # Create the KV cache.
62 - x = 16 // torch.tensor([], dtype=dtype).element_size()
63 - key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x, block_size, x)
64 - key_cache = torch.empty(size=key_cache_shape, dtype=dtype, device="cuda")
65 - key_cache.uniform_(-scale, scale)
66 - value_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size, block_size)
67 - value_cache = torch.empty(size=value_cache_shape,
68 - dtype=dtype,
69 - device="cuda")
70 - value_cache.uniform_(-scale, scale)
65 + key_caches, value_caches = create_kv_caches_with_random(
66 + NUM_BLOCKS, block_size, 1, num_kv_heads, head_size, kv_cache_dtype,
67 + dtype)
68 + key_cache, value_cache = key_caches[0], value_caches[0]
69
70 # Prepare for the paged attention kernel.
71 output = torch.empty_like(query)
@@ -106,6 +104,7 @@ def main(
104 block_size,
105 max_context_len,
106 alibi_slopes,
107 + kv_cache_dtype,
108 )
109 elif version == "v2":
110 ops.paged_attention_v2(
@@ -123,6 +122,7 @@ def main(
122 block_size,
123 max_context_len,
124 alibi_slopes,
125 + kv_cache_dtype,
126 )
127 else:
128 raise ValueError(f"Invalid version: {version}")
@@ -168,16 +168,18 @@ if __name__ == '__main__':
168 default="half")
169 parser.add_argument("--seed", type=int, default=0)
170 parser.add_argument("--profile", action="store_true")
171 + parser.add_argument(
172 + "--kv-cache-dtype",
173 + type=str,
174 + choices=["auto", "fp8_e5m2"],
175 + default="auto",
176 + help=
177 + 'Data type for kv cache storage. If "auto", will use model data type.')
178 args = parser.parse_args()
179 print(args)
180
181 if args.num_query_heads % args.num_kv_heads != 0:
182 raise ValueError("num_query_heads must be divisible by num_kv_heads")
176 - dtype_to_torch_dtype = {
177 - "half": torch.half,
178 - "bfloat16": torch.bfloat16,
179 - "float": torch.float,
180 - }
183 main(
184 version=args.version,
185 num_seqs=args.batch_size,
@@ -187,7 +189,8 @@ if __name__ == '__main__':
189 head_size=args.head_size,
190 block_size=args.block_size,
191 use_alibi=args.use_alibi,
190 - dtype=dtype_to_torch_dtype[args.dtype],
192 + dtype=STR_DTYPE_TO_TORCH_DTYPE[args.dtype],
193 seed=args.seed,
194 do_profile=args.profile,
195 + kv_cache_dtype=args.kv_cache_dtype,
196 )
csrc/attention/attention_dtypes.h
+1
@@ -4,3 +4,4 @@
4 #include "dtype_float16.cuh"
5 #include "dtype_float32.cuh"
6 #include "dtype_bfloat16.cuh"
7 +#include "dtype_fp8_e5m2.cuh"
csrc/attention/attention_kernels.cu
+164 -95
@@ -25,6 +25,7 @@
25
26 #include "attention_dtypes.h"
27 #include "attention_utils.cuh"
28 +#include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
29
30 #include <algorithm>
31
@@ -79,17 +80,19 @@ inline __device__ float block_sum(float* red_smem, float sum) {
80 // Grid: (num_heads, num_seqs, max_num_partitions).
81 template<
82 typename scalar_t,
83 + typename cache_t,
84 int HEAD_SIZE,
85 int BLOCK_SIZE,
86 int NUM_THREADS,
87 + bool IS_FP8_E5M2_KV_CACHE,
88 int PARTITION_SIZE = 0> // Zero means no partitioning.
89 __device__ void paged_attention_kernel(
90 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
91 float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
92 scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions, head_size]
93 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
91 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
92 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
94 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
95 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
96 const int num_kv_heads, // [num_heads]
97 const float scale,
98 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
@@ -145,6 +148,9 @@ __device__ void paged_attention_kernel(
148 constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
149 using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
150 using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
151 +#ifdef ENABLE_FP8_E5M2
152 + using Quant_vec = typename Vec<cache_t, VEC_SIZE>::Type;
153 +#endif
154
155 constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
156 constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
@@ -176,7 +182,7 @@ __device__ void paged_attention_kernel(
182
183 // x == THREAD_GROUP_SIZE * VEC_SIZE
184 // Each thread group fetches x elements from the key at a time.
179 - constexpr int x = 16 / sizeof(scalar_t);
185 + constexpr int x = 16 / sizeof(cache_t);
186 float qk_max = -FLT_MAX;
187
188 // Iterate over the key blocks.
@@ -202,13 +208,23 @@ __device__ void paged_attention_kernel(
208
209 #pragma unroll
210 for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
205 - const scalar_t* k_ptr = k_cache + physical_block_number * kv_block_stride
206 - + kv_head_idx * kv_head_stride
207 - + physical_block_offset * x;
211 + const cache_t* k_ptr = k_cache + physical_block_number * kv_block_stride
212 + + kv_head_idx * kv_head_stride
213 + + physical_block_offset * x;
214 const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
215 const int offset1 = (vec_idx * VEC_SIZE) / x;
216 const int offset2 = (vec_idx * VEC_SIZE) % x;
211 - k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
217 + if constexpr (IS_FP8_E5M2_KV_CACHE) {
218 +#ifdef ENABLE_FP8_E5M2
219 + Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
220 + // Vector conversion from Quant_vec to K_vec.
221 + k_vecs[j] = fp8_e5m2_unscaled::vec_conversion<K_vec, Quant_vec>(k_vec_quant);
222 +#else
223 + assert(false);
224 +#endif
225 + } else {
226 + k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
227 + }
228 }
229
230 // Compute dot product.
@@ -282,6 +298,9 @@ __device__ void paged_attention_kernel(
298 constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
299 using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
300 using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
301 +#ifdef ENABLE_FP8_E5M2
302 + using V_quant_vec = typename Vec<cache_t, V_VEC_SIZE>::Type;
303 +#endif
304 using Float_L_vec = typename FloatVec<L_vec>::Type;
305
306 constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
@@ -307,14 +326,25 @@ __device__ void paged_attention_kernel(
326 L_vec logits_vec;
327 from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx - start_token_idx));
328
310 - const scalar_t* v_ptr = v_cache + physical_block_number * kv_block_stride
311 - + kv_head_idx * kv_head_stride;
329 + const cache_t* v_ptr = v_cache + physical_block_number * kv_block_stride
330 + + kv_head_idx * kv_head_stride;
331 #pragma unroll
332 for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
333 const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
334 if (row_idx < HEAD_SIZE) {
335 const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
317 - V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
336 + V_vec v_vec;
337 + if constexpr (IS_FP8_E5M2_KV_CACHE) {
338 +#ifdef ENABLE_FP8_E5M2
339 + V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
340 + // Vector conversion from V_quant_vec to V_vec.
341 + v_vec = fp8_e5m2_unscaled::vec_conversion<V_vec, V_quant_vec>(v_quant_vec);
342 +#else
343 + assert(false);
344 +#endif
345 + } else {
346 + v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
347 + }
348 if (block_idx == num_context_blocks - 1) {
349 // NOTE(woosuk): When v_vec contains the tokens that are out of the context,
350 // we should explicitly zero out the values since they may contain NaNs.
@@ -395,14 +425,16 @@ __device__ void paged_attention_kernel(
425 // Grid: (num_heads, num_seqs, 1).
426 template<
427 typename scalar_t,
428 + typename cache_t,
429 int HEAD_SIZE,
430 int BLOCK_SIZE,
400 - int NUM_THREADS>
431 + int NUM_THREADS,
432 + bool IS_FP8_E5M2_KV_CACHE>
433 __global__ void paged_attention_v1_kernel(
434 scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
435 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
404 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
405 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
436 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
437 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
438 const int num_kv_heads, // [num_heads]
439 const float scale,
440 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
@@ -412,7 +444,7 @@ __global__ void paged_attention_v1_kernel(
444 const int q_stride,
445 const int kv_block_stride,
446 const int kv_head_stride) {
415 - paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
447 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE>(
448 /* exp_sums */ nullptr, /* max_logits */ nullptr,
449 out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
450 max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
@@ -421,17 +453,19 @@ __global__ void paged_attention_v1_kernel(
453 // Grid: (num_heads, num_seqs, max_num_partitions).
454 template<
455 typename scalar_t,
456 + typename cache_t,
457 int HEAD_SIZE,
458 int BLOCK_SIZE,
459 int NUM_THREADS,
460 + bool IS_FP8_E5M2_KV_CACHE,
461 int PARTITION_SIZE>
462 __global__ void paged_attention_v2_kernel(
463 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
464 float* __restrict__ max_logits, // [num_seqs, num_heads, max_num_partitions]
465 scalar_t* __restrict__ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
466 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
433 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
434 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
467 + const cache_t* __restrict__ k_cache, // [num_blocks, num_kv_heads, head_size/x, block_size, x]
468 + const cache_t* __restrict__ v_cache, // [num_blocks, num_kv_heads, head_size, block_size]
469 const int num_kv_heads, // [num_heads]
470 const float scale,
471 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
@@ -441,7 +475,7 @@ __global__ void paged_attention_v2_kernel(
475 const int q_stride,
476 const int kv_block_stride,
477 const int kv_head_stride) {
444 - paged_attention_kernel<scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE>(
478 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE>(
479 exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
480 block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
481 q_stride, kv_block_stride, kv_head_stride);
@@ -550,10 +584,10 @@ __global__ void paged_attention_v2_reduce_kernel(
584
585 #define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
586 VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
553 - ((void*)vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>), \
554 - shared_mem_size); \
555 - vllm::paged_attention_v1_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
556 - <<<grid, block, shared_mem_size, stream>>>( \
587 + ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
588 + IS_FP8_E5M2_KV_CACHE>), shared_mem_size); \
589 + vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
590 + IS_FP8_E5M2_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
591 out_ptr, \
592 query_ptr, \
593 key_cache_ptr, \
@@ -571,7 +605,9 @@ __global__ void paged_attention_v2_reduce_kernel(
605 // TODO(woosuk): Tune NUM_THREADS.
606 template<
607 typename T,
608 + typename CACHE_T,
609 int BLOCK_SIZE,
610 + bool IS_FP8_E5M2_KV_CACHE,
611 int NUM_THREADS = 128>
612 void paged_attention_v1_launcher(
613 torch::Tensor& out,
@@ -602,8 +638,8 @@ void paged_attention_v1_launcher(
638
639 T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
640 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
605 - T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
606 - T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
641 + CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
642 + CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
643 int* block_tables_ptr = block_tables.data_ptr<int>();
644 int* context_lens_ptr = context_lens.data_ptr<int>();
645
@@ -647,35 +683,35 @@ void paged_attention_v1_launcher(
683 }
684 }
685
650 -#define CALL_V1_LAUNCHER(T, BLOCK_SIZE) \
651 - paged_attention_v1_launcher<T, BLOCK_SIZE>( \
652 - out, \
653 - query, \
654 - key_cache, \
655 - value_cache, \
656 - num_kv_heads, \
657 - scale, \
658 - block_tables, \
659 - context_lens, \
660 - max_context_len, \
686 +#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
687 + paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
688 + out, \
689 + query, \
690 + key_cache, \
691 + value_cache, \
692 + num_kv_heads, \
693 + scale, \
694 + block_tables, \
695 + context_lens, \
696 + max_context_len, \
697 alibi_slopes);
698
699 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
700 // 1, 2, 4, 64, 128, 256.
665 -#define CALL_V1_LAUNCHER_BLOCK_SIZE(T) \
666 - switch (block_size) { \
667 - case 8: \
668 - CALL_V1_LAUNCHER(T, 8); \
669 - break; \
670 - case 16: \
671 - CALL_V1_LAUNCHER(T, 16); \
672 - break; \
673 - case 32: \
674 - CALL_V1_LAUNCHER(T, 32); \
675 - break; \
676 - default: \
677 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
678 - break; \
701 +#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
702 + switch (block_size) { \
703 + case 8: \
704 + CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
705 + break; \
706 + case 16: \
707 + CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
708 + break; \
709 + case 32: \
710 + CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
711 + break; \
712 + default: \
713 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
714 + break; \
715 }
716
717 void paged_attention_v1(
@@ -689,20 +725,36 @@ void paged_attention_v1(
725 torch::Tensor& context_lens, // [num_seqs]
726 int block_size,
727 int max_context_len,
692 - const c10::optional<torch::Tensor>& alibi_slopes) {
693 - if (query.dtype() == at::ScalarType::Float) {
694 - CALL_V1_LAUNCHER_BLOCK_SIZE(float);
695 - } else if (query.dtype() == at::ScalarType::Half) {
696 - CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t);
697 - } else if (query.dtype() == at::ScalarType::BFloat16) {
698 - CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
728 + const c10::optional<torch::Tensor>& alibi_slopes,
729 + const std::string& kv_cache_dtype) {
730 + if (kv_cache_dtype == "auto") {
731 + if (query.dtype() == at::ScalarType::Float) {
732 + CALL_V1_LAUNCHER_BLOCK_SIZE(float, float, false);
733 + } else if (query.dtype() == at::ScalarType::Half) {
734 + CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
735 + } else if (query.dtype() == at::ScalarType::BFloat16) {
736 + CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
737 + } else {
738 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
739 + }
740 + } else if (kv_cache_dtype == "fp8_e5m2") {
741 + if (query.dtype() == at::ScalarType::Float) {
742 + CALL_V1_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
743 + } else if (query.dtype() == at::ScalarType::Half) {
744 + CALL_V1_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
745 + } else if (query.dtype() == at::ScalarType::BFloat16) {
746 + CALL_V1_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
747 + } else {
748 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
749 + }
750 } else {
700 - TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
751 + TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
752 }
753 }
754
755 #define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
705 - vllm::paged_attention_v2_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, PARTITION_SIZE> \
756 + vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
757 + IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE> \
758 <<<grid, block, shared_mem_size, stream>>>( \
759 exp_sums_ptr, \
760 max_logits_ptr, \
@@ -730,7 +782,9 @@ void paged_attention_v1(
782
783 template<
784 typename T,
785 + typename CACHE_T,
786 int BLOCK_SIZE,
787 + bool IS_FP8_E5M2_KV_CACHE,
788 int NUM_THREADS = 128,
789 int PARTITION_SIZE = 512>
790 void paged_attention_v2_launcher(
@@ -768,8 +822,8 @@ void paged_attention_v2_launcher(
822 float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
823 T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
824 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
771 - T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
772 - T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
825 + CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
826 + CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
827 int* block_tables_ptr = block_tables.data_ptr<int>();
828 int* context_lens_ptr = context_lens.data_ptr<int>();
829
@@ -816,38 +870,38 @@ void paged_attention_v2_launcher(
870 }
871 }
872
819 -#define CALL_V2_LAUNCHER(T, BLOCK_SIZE) \
820 - paged_attention_v2_launcher<T, BLOCK_SIZE>( \
821 - out, \
822 - exp_sums, \
823 - max_logits, \
824 - tmp_out, \
825 - query, \
826 - key_cache, \
827 - value_cache, \
828 - num_kv_heads, \
829 - scale, \
830 - block_tables, \
831 - context_lens, \
832 - max_context_len, \
873 +#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
874 + paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
875 + out, \
876 + exp_sums, \
877 + max_logits, \
878 + tmp_out, \
879 + query, \
880 + key_cache, \
881 + value_cache, \
882 + num_kv_heads, \
883 + scale, \
884 + block_tables, \
885 + context_lens, \
886 + max_context_len, \
887 alibi_slopes);
888
889 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
890 // 1, 2, 4, 64, 128, 256.
837 -#define CALL_V2_LAUNCHER_BLOCK_SIZE(T) \
838 - switch (block_size) { \
839 - case 8: \
840 - CALL_V2_LAUNCHER(T, 8); \
841 - break; \
842 - case 16: \
843 - CALL_V2_LAUNCHER(T, 16); \
844 - break; \
845 - case 32: \
846 - CALL_V2_LAUNCHER(T, 32); \
847 - break; \
848 - default: \
849 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
850 - break; \
891 +#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
892 + switch (block_size) { \
893 + case 8: \
894 + CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
895 + break; \
896 + case 16: \
897 + CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
898 + break; \
899 + case 32: \
900 + CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
901 + break; \
902 + default: \
903 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
904 + break; \
905 }
906
907 void paged_attention_v2(
@@ -864,15 +918,30 @@ void paged_attention_v2(
918 torch::Tensor& context_lens, // [num_seqs]
919 int block_size,
920 int max_context_len,
867 - const c10::optional<torch::Tensor>& alibi_slopes) {
868 - if (query.dtype() == at::ScalarType::Float) {
869 - CALL_V2_LAUNCHER_BLOCK_SIZE(float);
870 - } else if (query.dtype() == at::ScalarType::Half) {
871 - CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t);
872 - } else if (query.dtype() == at::ScalarType::BFloat16) {
873 - CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16);
921 + const c10::optional<torch::Tensor>& alibi_slopes,
922 + const std::string& kv_cache_dtype) {
923 + if (kv_cache_dtype == "auto") {
924 + if (query.dtype() == at::ScalarType::Float) {
925 + CALL_V2_LAUNCHER_BLOCK_SIZE(float, float, false);
926 + } else if (query.dtype() == at::ScalarType::Half) {
927 + CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint16_t, false);
928 + } else if (query.dtype() == at::ScalarType::BFloat16) {
929 + CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, __nv_bfloat16, false);
930 + } else {
931 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
932 + }
933 + } else if (kv_cache_dtype == "fp8_e5m2") {
934 + if (query.dtype() == at::ScalarType::Float) {
935 + CALL_V2_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
936 + } else if (query.dtype() == at::ScalarType::Half) {
937 + CALL_V2_LAUNCHER_BLOCK_SIZE(uint16_t, uint8_t, true);
938 + } else if (query.dtype() == at::ScalarType::BFloat16) {
939 + CALL_V2_LAUNCHER_BLOCK_SIZE(__nv_bfloat16, uint8_t, true);
940 + } else {
941 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
942 + }
943 } else {
875 - TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
944 + TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
945 }
946 }
947
csrc/attention/dtype_fp8_e5m2.cuh new
+35
@@ -0,0 +1,35 @@
1 +#pragma once
2 +
3 +#include "attention_generic.cuh"
4 +
5 +#include <stdint.h>
6 +#ifdef ENABLE_FP8_E5M2
7 +#include <cuda_fp8.h>
8 +#endif
9 +
10 +namespace vllm {
11 +#ifdef ENABLE_FP8_E5M2
12 +// fp8 vector types for quantization of kv cache
13 +
14 +template<>
15 +struct Vec<uint8_t, 1> {
16 + using Type = uint8_t;
17 +};
18 +
19 +template<>
20 +struct Vec<uint8_t, 2> {
21 + using Type = uint16_t;
22 +};
23 +
24 +template<>
25 +struct Vec<uint8_t, 4> {
26 + using Type = uint32_t;
27 +};
28 +
29 +template<>
30 +struct Vec<uint8_t, 8> {
31 + using Type = uint2;
32 +};
33 +#endif // ENABLE_FP8_E5M2
34 +
35 +} // namespace vllm
csrc/cache.h
+7 -1
@@ -20,7 +20,8 @@ void reshape_and_cache(
20 torch::Tensor& value,
21 torch::Tensor& key_cache,
22 torch::Tensor& value_cache,
23 - torch::Tensor& slot_mapping);
23 + torch::Tensor& slot_mapping,
24 + const std::string& kv_cache_dtype);
25
26 void gather_cached_kv(
27 torch::Tensor& key,
@@ -28,3 +29,8 @@ void gather_cached_kv(
29 torch::Tensor& key_cache,
30 torch::Tensor& value_cache,
31 torch::Tensor& slot_mapping);
32 +
33 +// Just for unittest
34 +void convert_fp8_e5m2(
35 + torch::Tensor& src_cache,
36 + torch::Tensor& dst_cache);
csrc/cache_kernels.cu
+108 -27
@@ -4,6 +4,7 @@
4
5 #include "cuda_compat.h"
6 #include "dispatch_utils.h"
7 +#include "quantization/fp8_e5m2_kvcache/quant_utils.cuh"
8
9 #include <algorithm>
10 #include <cassert>
@@ -131,7 +132,7 @@ void copy_blocks(
132 dim3 block(std::min(1024, numel_per_block));
133 const at::cuda::OptionalCUDAGuard device_guard(cache_device);
134 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
134 - VLLM_DISPATCH_FLOATING_TYPES(
135 + VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
136 key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
137 vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
138 key_cache_ptrs_tensor.data_ptr<int64_t>(),
@@ -143,12 +144,12 @@ void copy_blocks(
144
145 namespace vllm {
146
146 -template<typename scalar_t>
147 +template<typename scalar_t, typename cache_t, bool is_fp8_e5m2_kv_cache>
148 __global__ void reshape_and_cache_kernel(
149 const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
150 const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
150 - scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
151 - scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
151 + cache_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
152 + cache_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
153 const int64_t* __restrict__ slot_mapping, // [num_tokens]
154 const int key_stride,
155 const int value_stride,
@@ -185,19 +186,45 @@ __global__ void reshape_and_cache_kernel(
186 + head_idx * head_size * block_size
187 + head_offset * block_size
188 + block_offset;
188 - key_cache[tgt_key_idx] = key[src_key_idx];
189 - value_cache[tgt_value_idx] = value[src_value_idx];
189 + scalar_t tgt_key = key[src_key_idx];
190 + scalar_t tgt_value = value[src_value_idx];
191 + if constexpr (is_fp8_e5m2_kv_cache) {
192 +#ifdef ENABLE_FP8_E5M2
193 + key_cache[tgt_key_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_key);
194 + value_cache[tgt_value_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_value);
195 +#else
196 + assert(false);
197 +#endif
198 + } else {
199 + key_cache[tgt_key_idx] = tgt_key;
200 + value_cache[tgt_value_idx] = tgt_value;
201 + }
202 }
203 }
204
205 } // namespace vllm
206
207 +#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
208 + vllm::reshape_and_cache_kernel<KV_T, CACHE_T, IS_FP8_E5M2_KV_CACHE><<<grid, block, 0, stream>>>( \
209 + reinterpret_cast<KV_T*>(key.data_ptr()), \
210 + reinterpret_cast<KV_T*>(value.data_ptr()), \
211 + reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
212 + reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
213 + slot_mapping.data_ptr<int64_t>(), \
214 + key_stride, \
215 + value_stride, \
216 + num_heads, \
217 + head_size, \
218 + block_size, \
219 + x);
220 +
221 void reshape_and_cache(
222 torch::Tensor& key, // [num_tokens, num_heads, head_size]
223 torch::Tensor& value, // [num_tokens, num_heads, head_size]
224 torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
225 torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
200 - torch::Tensor& slot_mapping) // [num_tokens]
226 + torch::Tensor& slot_mapping, // [num_tokens]
227 + const std::string& kv_cache_dtype)
228 {
229 int num_tokens = key.size(0);
230 int num_heads = key.size(1);
@@ -212,23 +239,25 @@ void reshape_and_cache(
239 dim3 block(std::min(num_heads * head_size, 512));
240 const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
241 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
215 - VLLM_DISPATCH_FLOATING_TYPES(
216 - key.scalar_type(),
217 - "reshape_and_cache_kernel",
218 - [&] {
219 - vllm::reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
220 - key.data_ptr<scalar_t>(),
221 - value.data_ptr<scalar_t>(),
222 - key_cache.data_ptr<scalar_t>(),
223 - value_cache.data_ptr<scalar_t>(),
224 - slot_mapping.data_ptr<int64_t>(),
225 - key_stride,
226 - value_stride,
227 - num_heads,
228 - head_size,
229 - block_size,
230 - x);
231 - });
242 + if (kv_cache_dtype == "auto") {
243 + if (key.dtype() == at::ScalarType::Float) {
244 + CALL_RESHAPE_AND_CACHE(float, float, false);
245 + } else if (key.dtype() == at::ScalarType::Half) {
246 + CALL_RESHAPE_AND_CACHE(uint16_t, uint16_t, false);
247 + } else if (key.dtype() == at::ScalarType::BFloat16) {
248 + CALL_RESHAPE_AND_CACHE(__nv_bfloat16, __nv_bfloat16, false);
249 + }
250 + } else if (kv_cache_dtype == "fp8_e5m2") {
251 + if (key.dtype() == at::ScalarType::Float) {
252 + CALL_RESHAPE_AND_CACHE(float, uint8_t, true);
253 + } else if (key.dtype() == at::ScalarType::Half) {
254 + CALL_RESHAPE_AND_CACHE(uint16_t, uint8_t, true);
255 + } else if (key.dtype() == at::ScalarType::BFloat16) {
256 + CALL_RESHAPE_AND_CACHE(__nv_bfloat16, uint8_t, true);
257 + }
258 + } else {
259 + TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
260 + }
261 }
262
263 namespace vllm {
@@ -256,12 +285,12 @@ __global__ void gather_cached_kv_kernel(
285 for (int i = threadIdx.x; i < num_tokens; i += blockDim.x) {
286 const int tgt_key_idx = token_idx * key_stride + i;
287 const int tgt_value_idx = token_idx * value_stride + i;
259 -
288 +
289 const int head_idx = i / head_size;
290 const int head_offset = i % head_size;
291 const int x_idx = head_offset / x; // the offset of the [head_size/x] dimension
292 const int x_offset = head_offset % x;
264 -
293 +
294 const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
295 + head_idx * (head_size / x) * block_size * x
296 + x_idx * block_size * x
@@ -373,7 +402,7 @@ void gather_cached_kv(
402 dim3 block(std::min(num_heads * head_size, 512));
403 const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
404 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
376 - VLLM_DISPATCH_FLOATING_TYPES(
405 + VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
406 key.scalar_type(),
407 "gather_cached_kv_kernel_optimized",
408 [&] {
@@ -391,3 +420,55 @@ void gather_cached_kv(
420 x);
421 });
422 }
423 +
424 +namespace vllm {
425 +
426 +template<typename Tout, typename Tin>
427 +__global__ void convert_fp8_e5m2_kernel(
428 + const Tin* __restrict__ src_cache,
429 + Tout* __restrict__ dst_cache,
430 + const int64_t block_stride) {
431 + const int64_t block_idx = blockIdx.x;
432 + for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
433 + int64_t idx = block_idx * block_stride + i;
434 +#ifdef ENABLE_FP8_E5M2
435 + dst_cache[idx] = fp8_e5m2_unscaled::vec_conversion<Tout, Tin>(src_cache[idx]);
436 +#else
437 + assert(false);
438 +#endif
439 + }
440 +}
441 +
442 +} // namespace vllm
443 +
444 +#define CALL_CONVERT_FP8_E5M2(Tout, Tin) \
445 + vllm::convert_fp8_e5m2_kernel<Tout, Tin><<<grid, block, 0, stream>>>( \
446 + reinterpret_cast<Tin*>(src_cache.data_ptr()), \
447 + reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
448 + block_stride);
449 +
450 +void convert_fp8_e5m2(
451 + torch::Tensor& src_cache,
452 + torch::Tensor& dst_cache)
453 +{
454 + int64_t num_blocks = src_cache.size(0);
455 + int64_t block_stride = src_cache.stride(0);
456 +
457 + dim3 grid(num_blocks);
458 + dim3 block(std::min(block_stride, int64_t(512)));
459 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
460 +
461 + if (src_cache.dtype() == at::ScalarType::Float) {
462 + CALL_CONVERT_FP8_E5M2(uint8_t, float);
463 + } else if (src_cache.dtype() == at::ScalarType::Half) {
464 + CALL_CONVERT_FP8_E5M2(uint8_t, uint16_t);
465 + } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
466 + CALL_CONVERT_FP8_E5M2(uint8_t, __nv_bfloat16);
467 + } else if (dst_cache.dtype() == at::ScalarType::Float) {
468 + CALL_CONVERT_FP8_E5M2(float, uint8_t);
469 + } else if (dst_cache.dtype() == at::ScalarType::Half) {
470 + CALL_CONVERT_FP8_E5M2(uint16_t, uint8_t);
471 + } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
472 + CALL_CONVERT_FP8_E5M2(__nv_bfloat16, uint8_t);
473 + }
474 +}
csrc/dispatch_utils.h
+10
@@ -14,3 +14,13 @@
14 #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15 AT_DISPATCH_SWITCH( \
16 TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
17 +
18 +#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
19 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
20 + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
21 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
22 + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__)
23 +
24 +#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \
25 + AT_DISPATCH_SWITCH( \
26 + TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__))
csrc/ops.h
+4 -2
@@ -13,7 +13,8 @@ void paged_attention_v1(
13 torch::Tensor& context_lens,
14 int block_size,
15 int max_context_len,
16 - const c10::optional<torch::Tensor>& alibi_slopes);
16 + const c10::optional<torch::Tensor>& alibi_slopes,
17 + const std::string& kv_cache_dtype);
18
19 void paged_attention_v2(
20 torch::Tensor& out,
@@ -29,7 +30,8 @@ void paged_attention_v2(
30 torch::Tensor& context_lens,
31 int block_size,
32 int max_context_len,
32 - const c10::optional<torch::Tensor>& alibi_slopes);
33 + const c10::optional<torch::Tensor>& alibi_slopes,
34 + const std::string& kv_cache_dtype);
35
36 void rms_norm(
37 torch::Tensor& out,
csrc/pybind.cpp
+4
@@ -75,6 +75,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
75 "gather_cached_kv",
76 &gather_cached_kv,
77 "Gather key and value from the cache into contiguous QKV tensors");
78 + cache_ops.def(
79 + "convert_fp8_e5m2",
80 + &convert_fp8_e5m2,
81 + "Convert the key and value cache to fp8_e5m2 data type");
82
83 // Cuda utils
84 pybind11::module cuda_utils = m.def_submodule("cuda_utils", "vLLM cuda utils");
csrc/quantization/fp8_e5m2_kvcache/quant_utils.cuh new
+278
@@ -0,0 +1,278 @@
1 +#pragma once
2 +
3 +#include <assert.h>
4 +#include <stdint.h>
5 +#include <float.h>
6 +#include <type_traits>
7 +#include "../../attention/attention_dtypes.h"
8 +#include "../../attention/dtype_float32.cuh"
9 +#include "../../attention/dtype_float16.cuh"
10 +#include "../../attention/dtype_bfloat16.cuh"
11 +
12 +#pragma once
13 +
14 +namespace vllm {
15 +#ifdef ENABLE_FP8_E5M2
16 +namespace fp8_e5m2_unscaled {
17 +
18 +template<typename Tout, typename Tin>
19 +__inline__ __device__ Tout vec_conversion(const Tin& x)
20 +{
21 + return x;
22 +}
23 +
24 +// fp8 -> half
25 +template<>
26 +__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(const uint8_t& a)
27 +{
28 + __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
29 + return res.x;
30 +}
31 +
32 +// fp8x2 -> half2
33 +template<>
34 +__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(const uint16_t& a)
35 +{
36 + union {
37 + uint16_t u16[2];
38 + uint32_t u32;
39 + } tmp;
40 + __half2_raw res = __nv_cvt_fp8x2_to_halfraw2(a, __NV_E5M2);
41 + tmp.u16[0] = res.x;
42 + tmp.u16[1] = res.y;
43 + return tmp.u32;
44 +}
45 +
46 +// fp8x4 -> half2x2
47 +template<>
48 +__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a)
49 +{
50 + union {
51 + uint2 u32x2;
52 + uint32_t u32[2];
53 + } tmp;
54 + tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
55 + tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
56 + return tmp.u32x2;
57 +}
58 +
59 +// fp8x8 -> half2x4
60 +template<>
61 +__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a)
62 +{
63 + union {
64 + uint4 u64x2;
65 + uint2 u64[2];
66 + } tmp;
67 + tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
68 + tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
69 + return tmp.u64x2;
70 +}
71 +
72 +// fp8 -> __nv_bfloat16
73 +template<>
74 +__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a)
75 +{
76 + // Note there is no direct convert function from fp8 to bf16.
77 + // fp8 -> half
78 + __half_raw res = __nv_cvt_fp8_to_halfraw(a, __NV_E5M2);
79 + // half -> float -> bf16
80 + float tmp = half_to_float(res.x);
81 + return __float2bfloat16(tmp);
82 +}
83 +
84 +// fp8x2 -> __nv_bfloat162
85 +template<>
86 +__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a)
87 +{
88 + __nv_bfloat162 res;
89 + res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
90 + res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
91 + return res;
92 +}
93 +
94 +// fp8x4 -> bf16_4_t
95 +template<>
96 +__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a)
97 +{
98 + bf16_4_t res;
99 + res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
100 + res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
101 + return res;
102 +}
103 +
104 +// fp8x8 -> bf16_8_t
105 +template<>
106 +__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a)
107 +{
108 + bf16_4_t tmp1, tmp2;
109 + tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
110 + tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
111 + bf16_8_t res;
112 + res.x = tmp1.x;
113 + res.y = tmp1.y;
114 + res.z = tmp2.x;
115 + res.w = tmp2.y;
116 + return res;
117 +}
118 +
119 +// fp8 -> float
120 +template<>
121 +__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a)
122 +{
123 + // fp8 -> half
124 + uint16_t tmp = vec_conversion<uint16_t, uint8_t>(a);
125 + // half -> float
126 + return half_to_float(tmp);
127 +}
128 +
129 +// fp8x2 -> float2
130 +template<>
131 +__inline__ __device__ float2 vec_conversion<float2, uint16_t>(const uint16_t& a)
132 +{
133 + // fp8x2 -> half2
134 + uint32_t tmp = vec_conversion<uint32_t, uint16_t>(a);
135 + // half2 -> float2
136 + return half2_to_float2(tmp);
137 +}
138 +
139 +// fp8x4 -> float4
140 +template<>
141 +__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(const uint32_t& a)
142 +{
143 + Float4_ res;
144 + res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
145 + res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
146 + return res;
147 +}
148 +
149 +// fp8x8 -> float8
150 +template<>
151 +__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a)
152 +{
153 + Float4_ tmp1, tmp2;
154 + tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
155 + tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
156 + Float8_ res;
157 + res.x = tmp1.x;
158 + res.y = tmp1.y;
159 + res.z = tmp2.x;
160 + res.w = tmp2.y;
161 + return res;
162 +}
163 +
164 +
165 +// half -> fp8
166 +template<>
167 +__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(const uint16_t& a)
168 +{
169 + __half_raw tmp;
170 + tmp.x = a;
171 + __nv_fp8_storage_t res = __nv_cvt_halfraw_to_fp8(tmp, __NV_SATFINITE, __NV_E5M2);
172 + return (uint8_t)res;
173 +}
174 +
175 +// bf16 -> fp8
176 +template<>
177 +__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a)
178 +{
179 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
180 + assert(false);
181 +#else
182 + __nv_fp8_storage_t res = __nv_cvt_bfloat16raw_to_fp8(__nv_bfloat16_raw(a), __NV_SATFINITE, __NV_E5M2);
183 + return (uint8_t)res;
184 +#endif
185 +}
186 +
187 +// float -> fp8
188 +template<>
189 +__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a)
190 +{
191 + __nv_fp8_storage_t res = __nv_cvt_float_to_fp8(a, __NV_SATFINITE, __NV_E5M2);
192 + return (uint8_t)res;
193 +}
194 +
195 +// fp8x4 -> float4
196 +template<>
197 +__inline__ __device__ float4 vec_conversion<float4, uint32_t>(const uint32_t& a)
198 +{
199 + Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
200 + float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
201 + return res;
202 +}
203 +
204 +
205 +template<>
206 +__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(const float2& a)
207 +{
208 + union {
209 + half2 float16;
210 + uint32_t uint32;
211 + };
212 +
213 + float16 = __float22half2_rn(a);
214 + return uint32;
215 +}
216 +
217 +template<>
218 +__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a)
219 +{
220 + uint2 b;
221 + float2 val;
222 + val.x = a.x.x;
223 + val.y = a.x.y;
224 + b.x = vec_conversion<uint32_t, float2>(val);
225 +
226 + val.x = a.y.x;
227 + val.y = a.y.y;
228 + b.y = vec_conversion<uint32_t, float2>(val);
229 +
230 + return b;
231 +}
232 +
233 +template<>
234 +__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a)
235 +{
236 + float4 b;
237 + b.x = a.x.x;
238 + b.y = a.x.y;
239 + b.z = a.y.x;
240 + b.w = a.y.y;
241 + return b;
242 +}
243 +
244 +template<>
245 +__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a)
246 +{
247 + uint4 b;
248 + b.x = vec_conversion<uint32_t, float2>(a.x);
249 + b.y = vec_conversion<uint32_t, float2>(a.y);
250 + b.z = vec_conversion<uint32_t, float2>(a.z);
251 + b.w = vec_conversion<uint32_t, float2>(a.w);
252 + return b;
253 +}
254 +
255 +template<>
256 +__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(const float2 &a) {
257 + __nv_bfloat162 b;
258 + from_float(b, a);
259 + return b;
260 +}
261 +
262 +template<>
263 +__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(const Float4_ &a) {
264 + bf16_4_t b;
265 + from_float(b, a);
266 + return b;
267 +}
268 +
269 +template<>
270 +__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(const Float8_ &a) {
271 + bf16_8_t b;
272 + from_float(b, a);
273 + return b;
274 +}
275 +
276 +} // namespace fp8_e5m2_unscaled
277 +#endif // ENABLE_FP8_E5M2
278 +} // namespace vllm
docs/source/quantization/fp8_e5m2_kv_cache.rst new
+32
@@ -0,0 +1,32 @@
1 +.. _fp8_e5m2_kv_cache:
2 +
3 +FP8 E5M2 KV Cache
4 +==================
5 +
6 +The int8/int4 quantization scheme requires additional scale GPU memory storage, which reduces the expected GPU memory benefits.
7 +The FP8 data format retains 2~3 mantissa bits and can convert float/fp16/bflaot16 and fp8 to each other.
8 +
9 +Here is an example of how to enable this feature:
10 +
11 +.. code-block:: python
12 + from vllm import LLM, SamplingParams
13 + # Sample prompts.
14 + prompts = [
15 + "Hello, my name is",
16 + "The president of the United States is",
17 + "The capital of France is",
18 + "The future of AI is",
19 + ]
20 + # Create a sampling params object.
21 + sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22 + # Create an LLM.
23 + llm = LLM(model="facebook/opt-125m", kv_cache_dtype="fp8_e5m2")
24 + # Generate texts from the prompts. The output is a list of RequestOutput objects
25 + # that contain the prompt, generated text, and other information.
26 + outputs = llm.generate(prompts, sampling_params)
27 + # Print the outputs.
28 + for output in outputs:
29 + prompt = output.prompt
30 + generated_text = output.outputs[0].text
31 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
32 +
setup.py
+3
@@ -253,6 +253,9 @@ if _is_cuda():
253 num_threads = min(os.cpu_count(), nvcc_threads)
254 NVCC_FLAGS += ["--threads", str(num_threads)]
255
256 + if nvcc_cuda_version >= Version("11.8"):
257 + NVCC_FLAGS += ["-DENABLE_FP8_E5M2"]
258 +
259 # changes for punica kernels
260 NVCC_FLAGS += torch_cpp_ext.COMMON_NVCC_FLAGS
261 REMOVE_NVCC_FLAGS = [
tests/kernels/conftest.py
+2 -39
@@ -1,44 +1,7 @@
1 -from typing import List, Tuple
2 -
1 import pytest
4 -import torch
5 -
6 -
7 -def create_kv_caches(
8 - num_blocks: int,
9 - block_size: int,
10 - num_layers: int,
11 - num_heads: int,
12 - head_size: int,
13 - dtype: torch.dtype,
14 - seed: int,
15 - device: str,
16 -) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
17 - torch.random.manual_seed(seed)
18 - torch.cuda.manual_seed(seed)
19 -
20 - scale = head_size**-0.5
21 - x = 16 // torch.tensor([], dtype=dtype).element_size()
22 - key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
23 - key_caches = []
24 - for _ in range(num_layers):
25 - key_cache = torch.empty(size=key_cache_shape,
26 - dtype=dtype,
27 - device=device)
28 - key_cache.uniform_(-scale, scale)
29 - key_caches.append(key_cache)
30 -
31 - value_cache_shape = (num_blocks, num_heads, head_size, block_size)
32 - value_caches = []
33 - for _ in range(num_layers):
34 - value_cache = torch.empty(size=value_cache_shape,
35 - dtype=dtype,
36 - device=device)
37 - value_cache.uniform_(-scale, scale)
38 - value_caches.append(value_cache)
39 - return key_caches, value_caches
2 +from vllm.utils import create_kv_caches_with_random
3
4
5 @pytest.fixture()
6 def kv_cache_factory():
44 - return create_kv_caches
7 + return create_kv_caches_with_random
tests/kernels/test_attention.py
+36 -5
@@ -6,14 +6,16 @@ import torch
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9 -from vllm._C import ops
9 +from vllm._C import ops, cache_ops
10 from vllm.utils import get_max_shared_memory_bytes
11
12 FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
13 # This will change depending on the compute capability.
14 # - 512 as a buffer
15 MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
16 -NUM_BLOCKS = 12000 # Arbitrary values for testing
16 +# There may not be enough gpu memory due to large NUM_BLOCKS.
17 +# Reduce NUM_BLOCKS when it happens.
18 +NUM_BLOCKS = 4321 # Arbitrary values for testing
19 PARTITION_SIZE = 512
20
21 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -23,6 +25,7 @@ NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
25 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
26 BLOCK_SIZES = [16, 32]
27 USE_ALIBI = [False, True]
28 +KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
29 SEEDS = [0]
30 DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
31
@@ -105,6 +108,7 @@ def ref_single_query_cached_kv_attention(
108 @pytest.mark.parametrize("use_alibi", USE_ALIBI)
109 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
110 @pytest.mark.parametrize("dtype", DTYPES)
111 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
112 @pytest.mark.parametrize("seed", SEEDS)
113 @pytest.mark.parametrize("device", DEVICES)
114 def test_paged_attention(
@@ -116,6 +120,7 @@ def test_paged_attention(
120 use_alibi: bool,
121 block_size: int,
122 dtype: torch.dtype,
123 + kv_cache_dtype: str,
124 seed: int,
125 device: int,
126 ) -> None:
@@ -158,8 +163,9 @@ def test_paged_attention(
163
164 # Create the KV caches.
165 key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
161 - num_kv_heads, head_size, dtype,
162 - seed, gpu_id)
166 + num_kv_heads, head_size,
167 + kv_cache_dtype, dtype, seed,
168 + gpu_id)
169 key_cache, value_cache = key_caches[0], value_caches[0]
170
171 # Call the paged attention kernel.
@@ -177,6 +183,7 @@ def test_paged_attention(
183 block_size,
184 max_context_len,
185 alibi_slopes,
186 + kv_cache_dtype,
187 )
188 elif version == "v2":
189 num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
@@ -209,11 +216,30 @@ def test_paged_attention(
216 block_size,
217 max_context_len,
218 alibi_slopes,
219 + kv_cache_dtype,
220 )
221 else:
222 raise AssertionError(f"Unknown version: {version}")
223
224 # Run the reference implementation.
225 + if kv_cache_dtype == "fp8_e5m2":
226 + # Convert cache data back to dtype.
227 + x = 16 // torch.tensor([], dtype=dtype).element_size()
228 + key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
229 + block_size, x)
230 + dequantized_key_cache = torch.empty(size=key_cache_shape,
231 + dtype=dtype,
232 + device=gpu_id)
233 + cache_ops.convert_fp8_e5m2(key_cache, dequantized_key_cache)
234 + key_cache = dequantized_key_cache
235 +
236 + value_cache_shape = value_cache.shape
237 + dequantized_value_cache = torch.empty(size=value_cache_shape,
238 + dtype=dtype,
239 + device=gpu_id)
240 + cache_ops.convert_fp8_e5m2(value_cache, dequantized_value_cache)
241 + value_cache = dequantized_value_cache
242 +
243 ref_output = torch.empty_like(query)
244 ref_single_query_cached_kv_attention(
245 ref_output,
@@ -230,7 +256,12 @@ def test_paged_attention(
256 # NOTE(woosuk): Due to the kernel-level differences in the two
257 # implementations, there is a small numerical difference in the two
258 # outputs. Thus, we use a relaxed tolerance for the test.
233 - assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
259 + # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
260 + # so we use a relaxed tolerance for the test.
261 + atol, rtol = 1e-3, 1e-5
262 + if kv_cache_dtype == "fp8_e5m2":
263 + atol, rtol = 1e-2, 1e-5
264 + assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
265
266
267 def ref_multi_query_kv_attention(
tests/kernels/test_cache.py
+7 -3
@@ -15,6 +15,7 @@ NUM_BLOCKS = [1024, 3600] # Arbitrary values for testing
15 NUM_MAPPINGS = [256] # Arbitrary values for testing
16 SEEDS = [0]
17 DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
18 +KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
19
20
21 @pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@@ -26,6 +27,7 @@ DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
27 @pytest.mark.parametrize("dtype", DTYPES)
28 @pytest.mark.parametrize("seed", SEEDS)
29 @pytest.mark.parametrize("device", DEVICES)
30 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
31 @torch.inference_mode()
32 def test_copy_blocks(
33 kv_cache_factory,
@@ -38,6 +40,7 @@ def test_copy_blocks(
40 dtype: torch.dtype,
41 seed: int,
42 device: int,
43 + kv_cache_dtype: str,
44 ) -> None:
45 random.seed(seed)
46 torch.random.manual_seed(seed)
@@ -59,7 +62,8 @@ def test_copy_blocks(
62 # Create the KV caches.
63 key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
64 num_layers, num_heads,
62 - head_size, dtype, seed, gpu_id)
65 + head_size, kv_cache_dtype,
66 + dtype, seed, gpu_id)
67
68 # Clone the KV caches.
69 cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
@@ -124,7 +128,7 @@ def test_reshape_and_cache(
128 # Create the KV caches.
129 key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
130 num_heads, head_size, dtype,
127 - seed, gpu_id)
131 + None, seed, gpu_id)
132 key_cache, value_cache = key_caches[0], value_caches[0]
133
134 # Clone the KV caches.
@@ -133,7 +137,7 @@ def test_reshape_and_cache(
137
138 # Call the reshape_and_cache kernel.
139 cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
136 - slot_mapping)
140 + slot_mapping, "auto")
141
142 # Run the reference implementation.
143 reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
vllm/config.py
+28 -1
@@ -1,13 +1,14 @@
1 from typing import Optional, Union, ClassVar
2 from dataclasses import dataclass
3 import os
4 +from packaging.version import Version
5
6 import torch
7 from transformers import PretrainedConfig
8
9 from vllm.logger import init_logger
10 from vllm.transformers_utils.config import get_config
10 -from vllm.utils import get_cpu_memory, is_hip
11 +from vllm.utils import get_cpu_memory, is_hip, get_nvcc_cuda_version
12
13 logger = init_logger(__name__)
14
@@ -275,6 +276,7 @@ class CacheConfig:
276 gpu_memory_utilization: Fraction of GPU memory to use for the
277 vLLM execution.
278 swap_space: Size of the CPU swap space per GPU (in GiB).
279 + cache_dtype: Data type for kv cache storage.
280 """
281
282 def __init__(
@@ -282,13 +284,16 @@ class CacheConfig:
284 block_size: int,
285 gpu_memory_utilization: float,
286 swap_space: int,
287 + cache_dtype: str,
288 sliding_window: Optional[int] = None,
289 ) -> None:
290 self.block_size = block_size
291 self.gpu_memory_utilization = gpu_memory_utilization
292 self.swap_space_bytes = swap_space * _GB
293 + self.cache_dtype = cache_dtype
294 self.sliding_window = sliding_window
295 self._verify_args()
296 + self._verify_cache_dtype()
297
298 # Will be set after profiling.
299 self.num_gpu_blocks = None
@@ -300,6 +305,28 @@ class CacheConfig:
305 "GPU memory utilization must be less than 1.0. Got "
306 f"{self.gpu_memory_utilization}.")
307
308 + def _verify_cache_dtype(self) -> None:
309 + if self.cache_dtype == "auto":
310 + pass
311 + elif self.cache_dtype == "fp8_e5m2":
312 + nvcc_cuda_version = get_nvcc_cuda_version()
313 + if nvcc_cuda_version < Version("11.8"):
314 + raise ValueError(
315 + "FP8 is not supported when cuda version is lower than 11.8."
316 + )
317 + device_name = torch.cuda.get_device_name()
318 + if "AMD" in device_name:
319 + raise NotImplementedError(
320 + "FP8_E5M2 KV Cache on AMD GPU has not been supported yet.")
321 + logger.info(
322 + "Using fp8_e5m2 data type to store kv cache. It reduces "
323 + "the GPU memory footprint and boosts the performance. "
324 + "But it may cause slight accuracy drop. "
325 + "Currently we only support fp8 without scaling factors and "
326 + "make e5m2 as a default format.")
327 + else:
328 + raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}")
329 +
330 def verify_with_parallel_config(
331 self,
332 parallel_config: "ParallelConfig",
vllm/engine/arg_utils.py
+10 -1
@@ -17,6 +17,7 @@ class EngineArgs:
17 download_dir: Optional[str] = None
18 load_format: str = 'auto'
19 dtype: str = 'auto'
20 + kv_cache_dtype: str = 'auto'
21 seed: int = 0
22 max_model_len: Optional[int] = None
23 worker_use_ray: bool = False
@@ -122,6 +123,14 @@ class EngineArgs:
123 'The "auto" option will use FP16 precision '
124 'for FP32 and FP16 models, and BF16 precision '
125 'for BF16 models.')
126 + parser.add_argument(
127 + '--kv-cache-dtype',
128 + type=str,
129 + choices=['auto', 'fp8_e5m2'],
130 + default='auto',
131 + help='Data type for kv cache storage. If "auto", will use model '
132 + 'data type. Note FP8 is not supported when cuda version is '
133 + 'lower than 11.8.')
134 parser.add_argument('--max-model-len',
135 type=int,
136 default=None,
@@ -269,7 +278,7 @@ class EngineArgs:
278 self.max_context_len_to_capture)
279 cache_config = CacheConfig(self.block_size,
280 self.gpu_memory_utilization,
272 - self.swap_space,
281 + self.swap_space, self.kv_cache_dtype,
282 model_config.get_sliding_window())
283 parallel_config = ParallelConfig(self.pipeline_parallel_size,
284 self.tensor_parallel_size,
vllm/engine/llm_engine.py
+6
@@ -85,6 +85,7 @@ class LLMEngine:
85 f"disable_custom_all_reduce={parallel_config.disable_custom_all_reduce}, "
86 f"quantization={model_config.quantization}, "
87 f"enforce_eager={model_config.enforce_eager}, "
88 + f"kv_cache_dtype={cache_config.cache_dtype}, "
89 f"seed={model_config.seed})")
90 # TODO(woosuk): Print more configs in debug mode.
91
@@ -144,6 +145,7 @@ class LLMEngine:
145 rank=0,
146 distributed_init_method=distributed_init_method,
147 lora_config=self.lora_config,
148 + kv_cache_dtype=self.cache_config.cache_dtype,
149 is_driver_worker=True,
150 )
151 self._run_workers("init_model")
@@ -234,6 +236,7 @@ class LLMEngine:
236 model_config = copy.deepcopy(self.model_config)
237 parallel_config = copy.deepcopy(self.parallel_config)
238 scheduler_config = copy.deepcopy(self.scheduler_config)
239 + cache_config = copy.deepcopy(self.cache_config)
240
241 for rank, (worker, (node_id,
242 _)) in enumerate(zip(self.workers,
@@ -249,6 +252,7 @@ class LLMEngine:
252 rank,
253 distributed_init_method,
254 lora_config=self.lora_config,
255 + cache_config=cache_config,
256 ))
257
258 driver_rank = 0
@@ -261,6 +265,7 @@ class LLMEngine:
265 driver_rank,
266 distributed_init_method,
267 lora_config=self.lora_config,
268 + cache_config=cache_config,
269 is_driver_worker=True,
270 )
271
@@ -306,6 +311,7 @@ class LLMEngine:
311 block_size=self.cache_config.block_size,
312 gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
313 cpu_swap_space=self.cache_config.swap_space_bytes,
314 + cache_dtype=self.cache_config.cache_dtype,
315 )
316
317 # Since we use a shared centralized controller, we take the minimum
vllm/model_executor/input_metadata.py
+5 -1
@@ -12,6 +12,7 @@ class InputMetadata:
12 max_context_len: The maximum context length.
13 context_lens: the length of attention context for each sequence.
14 block_tables: The block tables. (Seq id -> list of physical block)
15 + kv_cache_dtype: Data type to store kv cache.
16 """
17
18 def __init__(
@@ -25,6 +26,7 @@ class InputMetadata:
26 context_lens: Optional[torch.Tensor],
27 block_tables: Optional[torch.Tensor],
28 use_cuda_graph: bool,
29 + kv_cache_dtype: str,
30 ) -> None:
31 self.is_prompt = is_prompt
32 self.prompt_lens = prompt_lens
@@ -35,6 +37,7 @@ class InputMetadata:
37 self.context_lens = context_lens
38 self.block_tables = block_tables
39 self.use_cuda_graph = use_cuda_graph
40 + self.kv_cache_dtype = kv_cache_dtype
41
42 # Set during the execution of the first attention op.
43 # FIXME(woosuk): This is a hack.
@@ -47,4 +50,5 @@ class InputMetadata:
50 f"slot_mapping={self.slot_mapping}, "
51 f"context_lens={self.context_lens}, "
52 f"block_tables={self.block_tables}, "
50 - f"use_cuda_graph={self.use_cuda_graph})")
53 + f"use_cuda_graph={self.use_cuda_graph}, "
54 + f"kv_cache_dtype={self.kv_cache_dtype})")
vllm/model_executor/layers/attention.py
+3
@@ -98,6 +98,7 @@ class PagedAttention(nn.Module):
98 key_cache,
99 value_cache,
100 input_metadata.slot_mapping.flatten(),
101 + input_metadata.kv_cache_dtype,
102 )
103
104 if input_metadata.is_prompt:
@@ -265,6 +266,7 @@ def _paged_attention(
266 block_size,
267 input_metadata.max_context_len,
268 alibi_slopes,
269 + input_metadata.kv_cache_dtype,
270 )
271 else:
272 # Run PagedAttention V2.
@@ -295,5 +297,6 @@ def _paged_attention(
297 block_size,
298 input_metadata.max_context_len,
299 alibi_slopes,
300 + input_metadata.kv_cache_dtype,
301 )
302 return output
vllm/utils.py
+109 -1
@@ -1,9 +1,11 @@
1 import enum
2 import os
3 import socket
4 +import subprocess
5 import uuid
6 from platform import uname
6 -from typing import List
7 +from typing import List, Tuple, Union
8 +from packaging.version import parse, Version
9
10 import psutil
11 import torch
@@ -17,7 +19,17 @@ from typing import (
19 from collections import OrderedDict
20 from typing import Any, Hashable, Optional
21
22 +from vllm.logger import init_logger
23 +
24 T = TypeVar("T")
25 +logger = init_logger(__name__)
26 +
27 +STR_DTYPE_TO_TORCH_DTYPE = {
28 + "half": torch.half,
29 + "bfloat16": torch.bfloat16,
30 + "float": torch.float,
31 + "fp8_e5m2": torch.uint8,
32 +}
33
34
35 class Device(enum.Enum):
@@ -167,3 +179,99 @@ def get_open_port() -> int:
179
180 def set_cuda_visible_devices(device_ids: List[int]) -> None:
181 os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
182 +
183 +
184 +def get_nvcc_cuda_version() -> Version:
185 + cuda_home = os.environ.get('CUDA_HOME')
186 + if not cuda_home:
187 + cuda_home = '/usr/local/cuda'
188 + logger.info(
189 + f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
190 + )
191 + nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
192 + universal_newlines=True)
193 + output = nvcc_output.split()
194 + release_idx = output.index("release") + 1
195 + nvcc_cuda_version = parse(output[release_idx].split(",")[0])
196 + return nvcc_cuda_version
197 +
198 +
199 +def _generate_random_fp8_e5m2(
200 + tensor: torch.tensor,
201 + low: float,
202 + high: float,
203 +) -> None:
204 + # NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,
205 + # it may occur Inf or NaN if we directly use torch.randint
206 + # to generate random data for fp8 data.
207 + # For example, s.11111.00 in fp8e5m2 format repesents Inf.
208 + # | E4M3 | E5M2
209 + #-----|-------------|-------------------
210 + # Inf | N/A | s.11111.00
211 + # NaN | s.1111.111 | s.11111.{01,10,11}
212 + from vllm._C import cache_ops
213 + tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
214 + tensor_tmp.uniform_(low, high)
215 + cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)
216 + del tensor_tmp
217 +
218 +
219 +def create_kv_caches_with_random(
220 + num_blocks: int,
221 + block_size: int,
222 + num_layers: int,
223 + num_heads: int,
224 + head_size: int,
225 + cache_dtype: Optional[Union[str, torch.dtype]],
226 + model_dtype: Optional[Union[str, torch.dtype]] = None,
227 + seed: Optional[int] = 0,
228 + device: Optional[str] = "cuda",
229 +) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
230 + torch.random.manual_seed(seed)
231 + torch.cuda.manual_seed(seed)
232 +
233 + if isinstance(cache_dtype, str):
234 + if cache_dtype == "auto":
235 + if isinstance(model_dtype, str):
236 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]
237 + elif isinstance(model_dtype, torch.dtype):
238 + torch_dtype = model_dtype
239 + else:
240 + raise ValueError(f"Invalid model dtype: {model_dtype}")
241 + elif cache_dtype in ["half", "bfloat16", "float"]:
242 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
243 + elif cache_dtype == "fp8_e5m2":
244 + torch_dtype = torch.uint8
245 + else:
246 + raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
247 + elif isinstance(cache_dtype, torch.dtype):
248 + torch_dtype = cache_dtype
249 + else:
250 + raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
251 +
252 + scale = head_size**-0.5
253 + x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
254 + key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
255 + key_caches = []
256 + for _ in range(num_layers):
257 + key_cache = torch.empty(size=key_cache_shape,
258 + dtype=torch_dtype,
259 + device=device)
260 + if cache_dtype in ["auto", "half", "bfloat16", "float"]:
261 + key_cache.uniform_(-scale, scale)
262 + elif cache_dtype == 'fp8_e5m2':
263 + _generate_random_fp8_e5m2(key_cache, -scale, scale)
264 + key_caches.append(key_cache)
265 +
266 + value_cache_shape = (num_blocks, num_heads, head_size, block_size)
267 + value_caches = []
268 + for _ in range(num_layers):
269 + value_cache = torch.empty(size=value_cache_shape,
270 + dtype=torch_dtype,
271 + device=device)
272 + if cache_dtype in ["auto", "half", "bfloat16", "float"]:
273 + value_cache.uniform_(-scale, scale)
274 + elif cache_dtype == 'fp8_e5m2':
275 + _generate_random_fp8_e5m2(value_cache, -scale, scale)
276 + value_caches.append(value_cache)
277 + return key_caches, value_caches
vllm/worker/cache_engine.py
+12 -3
@@ -6,7 +6,7 @@ import torch
6 from vllm._C import cache_ops
7 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8 from vllm.logger import init_logger
9 -from vllm.utils import in_wsl
9 +from vllm.utils import in_wsl, STR_DTYPE_TO_TORCH_DTYPE
10
11 logger = init_logger(__name__)
12
@@ -34,12 +34,16 @@ class CacheEngine:
34 self.head_size = model_config.get_head_size()
35 self.num_layers = model_config.get_num_layers(parallel_config)
36 self.num_heads = model_config.get_num_kv_heads(parallel_config)
37 - self.dtype = model_config.dtype
37
38 self.block_size = cache_config.block_size
39 self.num_gpu_blocks = cache_config.num_gpu_blocks
40 self.num_cpu_blocks = cache_config.num_cpu_blocks
41
42 + if cache_config.cache_dtype == "auto":
43 + self.dtype = model_config.dtype
44 + else:
45 + self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
46 +
47 # Initialize the cache.
48 self.gpu_cache = self.allocate_gpu_cache()
49 self.cpu_cache = self.allocate_cpu_cache()
@@ -142,6 +146,7 @@ class CacheEngine:
146 @staticmethod
147 def get_cache_block_size(
148 block_size: int,
149 + cache_dtype: str,
150 model_config: ModelConfig,
151 parallel_config: ParallelConfig,
152 ) -> int:
@@ -152,7 +157,11 @@ class CacheEngine:
157 key_cache_block = block_size * num_heads * head_size
158 value_cache_block = key_cache_block
159 total = num_layers * (key_cache_block + value_cache_block)
155 - dtype_size = _get_dtype_size(model_config.dtype)
160 + if cache_dtype == "auto":
161 + dtype = model_config.dtype
162 + else:
163 + dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
164 + dtype_size = _get_dtype_size(dtype)
165 return dtype_size * total
166
167
vllm/worker/model_runner.py
+7
@@ -36,6 +36,7 @@ class ModelRunner:
36 parallel_config: ParallelConfig,
37 scheduler_config: SchedulerConfig,
38 lora_config: Optional[LoRAConfig],
39 + kv_cache_dtype: Optional[str] = "auto",
40 is_driver_worker: bool = False,
41 ):
42 self.model_config = model_config
@@ -68,6 +69,7 @@ class ModelRunner:
69 self.graph_block_tables = None # Set after initial profiling.
70 # cache in_wsl result
71 self.in_wsl = in_wsl()
72 + self.kv_cache_dtype = kv_cache_dtype
73
74 def load_model(self) -> None:
75 self.model = get_model(self.model_config, self.lora_config)
@@ -223,6 +225,7 @@ class ModelRunner:
225 context_lens=context_lens_tensor,
226 block_tables=block_tables,
227 use_cuda_graph=False,
228 + kv_cache_dtype=self.kv_cache_dtype,
229 )
230 return (input_tokens, input_positions, input_metadata, prompt_lens,
231 subquery_lens, lora_index_mapping, lora_prompt_mapping,
@@ -350,6 +353,7 @@ class ModelRunner:
353 context_lens=context_lens,
354 block_tables=block_tables,
355 use_cuda_graph=use_captured_graph,
356 + kv_cache_dtype=self.kv_cache_dtype,
357 )
358 return input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, lora_requests
359
@@ -473,6 +477,7 @@ class ModelRunner:
477 "context_lens": input_metadata.context_lens,
478 "block_tables": input_metadata.block_tables,
479 "use_cuda_graph": input_metadata.use_cuda_graph,
480 + "kv_cache_dtype": input_metadata.kv_cache_dtype,
481 "selected_token_indices":
482 sampling_metadata.selected_token_indices,
483 "lora_requests": lora_requests,
@@ -495,6 +500,7 @@ class ModelRunner:
500 context_lens=metadata_dict["context_lens"],
501 block_tables=metadata_dict["block_tables"],
502 use_cuda_graph=metadata_dict["use_cuda_graph"],
503 + kv_cache_dtype=metadata_dict["kv_cache_dtype"],
504 )
505 sampling_metadata = SamplingMetadata(
506 seq_groups=None,
@@ -665,6 +671,7 @@ class ModelRunner:
671 context_lens=context_lens[:batch_size],
672 block_tables=block_tables[:batch_size],
673 use_cuda_graph=True,
674 + kv_cache_dtype=self.kv_cache_dtype,
675 )
676
677 if self.lora_config:
vllm/worker/worker.py
+4 -1
@@ -37,6 +37,7 @@ class Worker:
37 rank: int,
38 distributed_init_method: str,
39 lora_config: Optional[LoRAConfig] = None,
40 + kv_cache_dtype: Optional[str] = "auto",
41 is_driver_worker: bool = False,
42 ) -> None:
43 self.model_config = model_config
@@ -54,6 +55,7 @@ class Worker:
55 parallel_config,
56 scheduler_config,
57 lora_config=self.lora_config,
58 + kv_cache_dtype=kv_cache_dtype,
59 is_driver_worker=is_driver_worker)
60 # Uninitialized cache engine. Will be initialized by
61 # self.init_cache_engine().
@@ -95,6 +97,7 @@ class Worker:
97 block_size: int,
98 gpu_memory_utilization: float,
99 cpu_swap_space: int,
100 + cache_dtype: str,
101 ) -> Tuple[int, int]:
102 """Profiles the peak memory usage of the model and returns the maximum
103 number of GPU and CPU cache blocks that can be allocated.
@@ -119,7 +122,7 @@ class Worker:
122 peak_memory = total_gpu_memory - free_gpu_memory
123
124 cache_block_size = CacheEngine.get_cache_block_size(
122 - block_size, self.model_config, self.parallel_config)
125 + block_size, cache_dtype, self.model_config, self.parallel_config)
126 num_gpu_blocks = int(
127 (total_gpu_memory * gpu_memory_utilization - peak_memory) //
128 cache_block_size)