@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 978aed5300
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Kernel][Attention] Separate `Attention.kv_scale` into `k_scale` and `v_scale` (#6081)

Michael Goin committed Jul 16, 2024 at 18:31 UTC 978aed53004b82877bd2af0f10afff1826d7194d
33 files changed +317 -185
benchmarks/kernels/benchmark_paged_attention.py
+5 -3
@@ -100,7 +100,7 @@ def main(
100 start_time = time.perf_counter()
101
102 # Using default kv_scale
103 - kv_scale = 1.0
103 + k_scale = v_scale = 1.0
104
105 for _ in range(num_iters):
106 if version == "v1":
@@ -117,7 +117,8 @@ def main(
117 max_seq_len,
118 alibi_slopes,
119 kv_cache_dtype,
120 - kv_scale,
120 + k_scale,
121 + v_scale,
122 )
123 elif version == "v2":
124 ops.paged_attention_v2(
@@ -136,7 +137,8 @@ def main(
137 max_seq_len,
138 alibi_slopes,
139 kv_cache_dtype,
139 - kv_scale,
140 + k_scale,
141 + v_scale,
142 )
143 else:
144 raise ValueError(f"Invalid version: {version}")
csrc/attention/attention_kernels.cu
+27 -26
@@ -105,9 +105,9 @@ __device__ void paged_attention_kernel(
105 const int max_num_blocks_per_seq,
106 const float* __restrict__ alibi_slopes, // [num_heads]
107 const int q_stride, const int kv_block_stride, const int kv_head_stride,
108 - const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
109 - const int blocksparse_vert_stride, const int blocksparse_block_size,
110 - const int blocksparse_head_sliding_step) {
108 + const float k_scale, const float v_scale, const int tp_rank,
109 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
110 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
111 const int seq_idx = blockIdx.y;
112 const int partition_idx = blockIdx.z;
113 const int max_num_partitions = gridDim.z;
@@ -285,7 +285,7 @@ __device__ void paged_attention_kernel(
285 Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
286 k_ptr + offset1 * BLOCK_SIZE * x + offset2);
287 k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(
288 - k_vec_quant, kv_scale);
288 + k_vec_quant, k_scale);
289 }
290 }
291
@@ -415,7 +415,7 @@ __device__ void paged_attention_kernel(
415 *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
416 // Vector conversion from V_quant_vec to V_vec.
417 v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec,
418 - kv_scale);
418 + v_scale);
419 }
420 if (block_idx == num_seq_blocks - 1) {
421 // NOTE(woosuk): When v_vec contains the tokens that are out of the
@@ -513,15 +513,15 @@ __global__ void paged_attention_v1_kernel(
513 const int max_num_blocks_per_seq,
514 const float* __restrict__ alibi_slopes, // [num_heads]
515 const int q_stride, const int kv_block_stride, const int kv_head_stride,
516 - const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
517 - const int blocksparse_vert_stride, const int blocksparse_block_size,
518 - const int blocksparse_head_sliding_step) {
516 + const float k_scale, const float v_scale, const int tp_rank,
517 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
518 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
519 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
520 KV_DTYPE, IS_BLOCK_SPARSE>(
521 /* exp_sums */ nullptr, /* max_logits */ nullptr, out, q, k_cache,
522 v_cache, num_kv_heads, scale, block_tables, seq_lens,
523 max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride,
524 - kv_head_stride, kv_scale, tp_rank, blocksparse_local_blocks,
524 + kv_head_stride, k_scale, v_scale, tp_rank, blocksparse_local_blocks,
525 blocksparse_vert_stride, blocksparse_block_size,
526 blocksparse_head_sliding_step);
527 }
@@ -549,14 +549,14 @@ __global__ void paged_attention_v2_kernel(
549 const int max_num_blocks_per_seq,
550 const float* __restrict__ alibi_slopes, // [num_heads]
551 const int q_stride, const int kv_block_stride, const int kv_head_stride,
552 - const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
553 - const int blocksparse_vert_stride, const int blocksparse_block_size,
554 - const int blocksparse_head_sliding_step) {
552 + const float k_scale, const float v_scale, const int tp_rank,
553 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
554 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
555 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
556 KV_DTYPE, IS_BLOCK_SPARSE, PARTITION_SIZE>(
557 exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
558 block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes, q_stride,
559 - kv_block_stride, kv_head_stride, kv_scale, tp_rank,
559 + kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank,
560 blocksparse_local_blocks, blocksparse_vert_stride, blocksparse_block_size,
561 blocksparse_head_sliding_step);
562 }
@@ -682,7 +682,7 @@ __global__ void paged_attention_v2_reduce_kernel(
682 out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
683 scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
684 alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
685 - kv_scale, tp_rank, blocksparse_local_blocks, \
685 + k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
686 blocksparse_vert_stride, blocksparse_block_size, \
687 blocksparse_head_sliding_step);
688
@@ -694,8 +694,8 @@ void paged_attention_v1_launcher(
694 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
695 torch::Tensor& value_cache, int num_kv_heads, float scale,
696 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
697 - const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
698 - const int tp_rank, const int blocksparse_local_blocks,
697 + const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
698 + float v_scale, const int tp_rank, const int blocksparse_local_blocks,
699 const int blocksparse_vert_stride, const int blocksparse_block_size,
700 const int blocksparse_head_sliding_step) {
701 int num_seqs = query.size(0);
@@ -770,7 +770,7 @@ void paged_attention_v1_launcher(
770 paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
771 IS_BLOCK_SPARSE>( \
772 out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
773 - seq_lens, max_seq_len, alibi_slopes, kv_scale, tp_rank, \
773 + seq_lens, max_seq_len, alibi_slopes, k_scale, v_scale, tp_rank, \
774 blocksparse_local_blocks, blocksparse_vert_stride, \
775 blocksparse_block_size, blocksparse_head_sliding_step);
776
@@ -815,8 +815,8 @@ void paged_attention_v1(
815 torch::Tensor& seq_lens, // [num_seqs]
816 int64_t block_size, int64_t max_seq_len,
817 const c10::optional<torch::Tensor>& alibi_slopes,
818 - const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
819 - const int64_t blocksparse_local_blocks,
818 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
819 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
820 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
821 const int64_t blocksparse_head_sliding_step) {
822 const bool is_block_sparse = (blocksparse_vert_stride > 1);
@@ -833,7 +833,7 @@ void paged_attention_v1(
833 exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
834 value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
835 seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
836 - kv_block_stride, kv_head_stride, kv_scale, tp_rank, \
836 + kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
837 blocksparse_local_blocks, blocksparse_vert_stride, \
838 blocksparse_block_size, blocksparse_head_sliding_step); \
839 vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
@@ -850,8 +850,8 @@ void paged_attention_v2_launcher(
850 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
851 torch::Tensor& value_cache, int num_kv_heads, float scale,
852 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
853 - const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
854 - const int tp_rank, const int blocksparse_local_blocks,
853 + const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
854 + float v_scale, const int tp_rank, const int blocksparse_local_blocks,
855 const int blocksparse_vert_stride, const int blocksparse_block_size,
856 const int blocksparse_head_sliding_step) {
857 int num_seqs = query.size(0);
@@ -932,8 +932,9 @@ void paged_attention_v2_launcher(
932 IS_BLOCK_SPARSE>( \
933 out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
934 num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
935 - kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride, \
936 - blocksparse_block_size, blocksparse_head_sliding_step);
935 + k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
936 + blocksparse_vert_stride, blocksparse_block_size, \
937 + blocksparse_head_sliding_step);
938
939 #define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
940 switch (is_block_sparse) { \
@@ -980,8 +981,8 @@ void paged_attention_v2(
981 torch::Tensor& seq_lens, // [num_seqs]
982 int64_t block_size, int64_t max_seq_len,
983 const c10::optional<torch::Tensor>& alibi_slopes,
983 - const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
984 - const int64_t blocksparse_local_blocks,
984 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
985 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
986 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
987 const int64_t blocksparse_head_sliding_step) {
988 const bool is_block_sparse = (blocksparse_vert_stride > 1);
csrc/cache.h
+2 -2
@@ -18,8 +18,8 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
18 void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
19 torch::Tensor& key_cache, torch::Tensor& value_cache,
20 torch::Tensor& slot_mapping,
21 - const std::string& kv_cache_dtype,
22 - const double kv_scale);
21 + const std::string& kv_cache_dtype, const double k_scale,
22 + const double v_scale);
23
24 void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
25 torch::Tensor& key_cache,
csrc/cache_kernels.cu
+11 -10
@@ -159,8 +159,8 @@ __global__ void reshape_and_cache_kernel(
159 // block_size]
160 const int64_t* __restrict__ slot_mapping, // [num_tokens]
161 const int key_stride, const int value_stride, const int num_heads,
162 - const int head_size, const int block_size, const int x,
163 - const float kv_scale) {
162 + const int head_size, const int block_size, const int x, const float k_scale,
163 + const float v_scale) {
164 const int64_t token_idx = blockIdx.x;
165 const int64_t slot_idx = slot_mapping[token_idx];
166 if (slot_idx < 0) {
@@ -196,9 +196,9 @@ __global__ void reshape_and_cache_kernel(
196 value_cache[tgt_value_idx] = tgt_value;
197 } else {
198 key_cache[tgt_key_idx] =
199 - fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
199 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, k_scale);
200 value_cache[tgt_value_idx] =
201 - fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
201 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, v_scale);
202 }
203 }
204 }
@@ -248,7 +248,7 @@ __global__ void reshape_and_cache_flash_kernel(
248 reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
249 reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
250 slot_mapping.data_ptr<int64_t>(), key_stride, value_stride, \
251 - num_heads, head_size, block_size, x, kv_scale);
251 + num_heads, head_size, block_size, x, k_scale, v_scale);
252
253 void reshape_and_cache(
254 torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -258,7 +258,8 @@ void reshape_and_cache(
258 torch::Tensor&
259 value_cache, // [num_blocks, num_heads, head_size, block_size]
260 torch::Tensor& slot_mapping, // [num_tokens]
261 - const std::string& kv_cache_dtype, const double kv_scale) {
261 + const std::string& kv_cache_dtype, const double k_scale,
262 + const double v_scale) {
263 int num_tokens = key.size(0);
264 int num_heads = key.size(1);
265 int head_size = key.size(2);
@@ -318,13 +319,13 @@ namespace vllm {
319 template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
320 __global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
321 Tout* __restrict__ dst_cache,
321 - const float kv_scale,
322 + const float scale,
323 const int64_t block_stride) {
324 const int64_t block_idx = blockIdx.x;
325 for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
326 int64_t idx = block_idx * block_stride + i;
327 dst_cache[idx] =
327 - fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
328 + fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], scale);
329 }
330 }
331
@@ -333,11 +334,11 @@ __global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
334 #define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
335 vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
336 reinterpret_cast<Tin*>(src_cache.data_ptr()), \
336 - reinterpret_cast<Tout*>(dst_cache.data_ptr()), kv_scale, block_stride);
337 + reinterpret_cast<Tout*>(dst_cache.data_ptr()), scale, block_stride);
338
339 // Only for testing.
340 void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
340 - const double kv_scale, const std::string& kv_cache_dtype) {
341 + const double scale, const std::string& kv_cache_dtype) {
342 torch::Device src_device = src_cache.device();
343 torch::Device dst_device = dst_cache.device();
344 TORCH_CHECK(src_device.is_cuda(), "src must be on a GPU")
csrc/cpu/attention.cpp
+6 -6
@@ -423,11 +423,11 @@ void paged_attention_v1(
423 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
424 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
425 int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
426 - const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
427 - const int64_t blocksparse_local_blocks,
426 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
427 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
428 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
429 const int64_t blocksparse_head_sliding_step) {
430 - TORCH_CHECK(kv_scale == 1.0f);
430 + TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
431 TORCH_CHECK(blocksparse_vert_stride <= 1,
432 "CPU backend does not support blocksparse attention yet.");
433 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
@@ -742,11 +742,11 @@ void paged_attention_v2(
742 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
743 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
744 int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
745 - const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
746 - const int64_t blocksparse_local_blocks,
745 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
746 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
747 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
748 const int64_t blocksparse_head_sliding_step) {
749 - TORCH_CHECK(kv_scale == 1.0f);
749 + TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
750 TORCH_CHECK(blocksparse_vert_stride <= 1,
751 "CPU backend does not support blocksparse attention yet.");
752 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
csrc/cpu/cache.cpp
+3 -2
@@ -107,8 +107,9 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
107 void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
108 torch::Tensor& key_cache, torch::Tensor& value_cache,
109 torch::Tensor& slot_mapping,
110 - const std::string& kv_cache_dtype, double kv_scale) {
111 - TORCH_CHECK(kv_scale == 1.0f);
110 + const std::string& kv_cache_dtype, double k_scale,
111 + double v_scale) {
112 + TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
113
114 int num_tokens = key.size(0);
115 int num_heads = key.size(1);
csrc/cpu/torch_bindings.cpp
+5 -5
@@ -16,8 +16,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
16 " Tensor value_cache, int num_kv_heads, float scale,"
17 " Tensor block_tables, Tensor seq_lens, int block_size,"
18 " int max_seq_len, Tensor? alibi_slopes,"
19 - " str kv_cache_dtype, float kv_scale, int tp_rank,"
20 - " int blocksparse_local_blocks,"
19 + " str kv_cache_dtype, float k_scale, float v_scale,"
20 + " int tp_rank, int blocksparse_local_blocks,"
21 " int blocksparse_vert_stride, int blocksparse_block_size,"
22 " int blocksparse_head_sliding_step) -> ()");
23 ops.impl("paged_attention_v1", torch::kCPU, &paged_attention_v1);
@@ -30,8 +30,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
30 " Tensor value_cache, int num_kv_heads, float scale,"
31 " Tensor block_tables, Tensor seq_lens, int block_size,"
32 " int max_seq_len, Tensor? alibi_slopes,"
33 - " str kv_cache_dtype, float kv_scale, int tp_rank,"
34 - " int blocksparse_local_blocks,"
33 + " str kv_cache_dtype, float k_scale, float v_scale,"
34 + " int tp_rank, int blocksparse_local_blocks,"
35 " int blocksparse_vert_stride, int blocksparse_block_size,"
36 " int blocksparse_head_sliding_step) -> ()");
37 ops.impl("paged_attention_v2", torch::kCPU, &paged_attention_v2);
@@ -103,7 +103,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
103 " Tensor! key_cache, Tensor! value_cache,"
104 " Tensor slot_mapping,"
105 " str kv_cache_dtype,"
106 - " float kv_scale) -> ()");
106 + " float k_scale, float v_scale) -> ()");
107 cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
108 }
109
csrc/ops.h
+4 -4
@@ -8,8 +8,8 @@ void paged_attention_v1(
8 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
9 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
10 int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
11 - const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
12 - const int64_t blocksparse_local_blocks,
11 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
12 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
13 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
14 const int64_t blocksparse_head_sliding_step);
15
@@ -19,8 +19,8 @@ void paged_attention_v2(
19 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
20 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
21 int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
22 - const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
23 - const int64_t blocksparse_local_blocks,
22 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
23 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
24 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
25 const int64_t blocksparse_head_sliding_step);
26
csrc/torch_bindings.cpp
+5 -5
@@ -27,8 +27,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
27 " Tensor value_cache, int num_kv_heads, float scale,"
28 " Tensor block_tables, Tensor seq_lens, int block_size,"
29 " int max_seq_len, Tensor? alibi_slopes,"
30 - " str kv_cache_dtype, float kv_scale, int tp_rank,"
31 - " int blocksparse_local_blocks,"
30 + " str kv_cache_dtype, float k_scale, float v_scale,"
31 + " int tp_rank, int blocksparse_local_blocks,"
32 " int blocksparse_vert_stride, int blocksparse_block_size,"
33 " int blocksparse_head_sliding_step) -> ()");
34 ops.impl("paged_attention_v1", torch::kCUDA, &paged_attention_v1);
@@ -41,8 +41,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
41 " Tensor value_cache, int num_kv_heads, float scale,"
42 " Tensor block_tables, Tensor seq_lens, int block_size,"
43 " int max_seq_len, Tensor? alibi_slopes,"
44 - " str kv_cache_dtype, float kv_scale, int tp_rank,"
45 - " int blocksparse_local_blocks,"
44 + " str kv_cache_dtype, float k_scale, float v_scale,"
45 + " int tp_rank, int blocksparse_local_blocks,"
46 " int blocksparse_vert_stride, int blocksparse_block_size,"
47 " int blocksparse_head_sliding_step) -> ()");
48 ops.impl("paged_attention_v2", torch::kCUDA, &paged_attention_v2);
@@ -223,7 +223,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
223 " Tensor! key_cache, Tensor! value_cache,"
224 " Tensor slot_mapping,"
225 " str kv_cache_dtype,"
226 - " float kv_scale) -> ()");
226 + " float k_scale, float v_scale) -> ()");
227 cache_ops.impl("reshape_and_cache", torch::kCUDA, &reshape_and_cache);
228
229 // Reshape the key and value tensors and cache them.
tests/kernels/test_attention.py
+5 -3
@@ -175,7 +175,7 @@ def test_paged_attention(
175 key_cache, value_cache = key_caches[0], value_caches[0]
176
177 # Using default kv_scale
178 - kv_scale = 1.0
178 + k_scale = v_scale = 1.0
179
180 # Call the paged attention kernel.
181 output = torch.empty_like(query)
@@ -193,7 +193,8 @@ def test_paged_attention(
193 max_seq_len,
194 alibi_slopes,
195 kv_cache_dtype,
196 - kv_scale,
196 + k_scale,
197 + v_scale,
198 )
199 elif version == "v2":
200 num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
@@ -224,7 +225,8 @@ def test_paged_attention(
225 max_seq_len,
226 alibi_slopes,
227 kv_cache_dtype,
227 - kv_scale,
228 + k_scale,
229 + v_scale,
230 )
231 else:
232 raise AssertionError(f"Unknown version: {version}")
tests/kernels/test_blocksparse_attention.py
+5 -3
@@ -212,7 +212,7 @@ def test_paged_attention(
212 key_cache, value_cache = key_caches[0], value_caches[0]
213
214 # Using default kv_scale
215 - kv_scale = 1.0
215 + k_scale = v_scale = 1.0
216 tp_rank = 0
217
218 # Call the paged attention kernel.
@@ -231,7 +231,8 @@ def test_paged_attention(
231 max_seq_len,
232 alibi_slopes,
233 kv_cache_dtype,
234 - kv_scale,
234 + k_scale,
235 + v_scale,
236 tp_rank=tp_rank,
237 blocksparse_local_blocks=blocksparse_local_blocks,
238 blocksparse_vert_stride=blocksparse_vert_stride,
@@ -267,7 +268,8 @@ def test_paged_attention(
268 max_seq_len,
269 alibi_slopes,
270 kv_cache_dtype,
270 - kv_scale,
271 + k_scale,
272 + v_scale,
273 tp_rank=tp_rank,
274 blocksparse_local_blocks=blocksparse_local_blocks,
275 blocksparse_vert_stride=blocksparse_vert_stride,
tests/kernels/test_cache.py
+2 -2
@@ -155,11 +155,11 @@ def test_reshape_and_cache(
155 cloned_value_cache = value_cache.clone()
156
157 # Using default kv_scale
158 - kv_scale = 1.0
158 + k_scale = v_scale = 1.0
159
160 # Call the reshape_and_cache kernel.
161 ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping,
162 - kv_cache_dtype, kv_scale)
162 + kv_cache_dtype, k_scale, v_scale)
163
164 if kv_cache_dtype == "fp8":
165 result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
tests/quantization/test_fp8.py
+35 -5
@@ -7,19 +7,49 @@ import torch
7
8 from tests.quantization.utils import is_quant_method_supported
9 from vllm import _custom_ops as ops
10 -from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
10 +from vllm.model_executor.layers.quantization.fp8 import (Fp8KVCacheMethod,
11 + Fp8LinearMethod)
12
13 MODELS = [
13 - "neuralmagic/Meta-Llama-3-8B-Instruct-FP8",
14 + "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
15 "nm-testing/Phi-3-mini-128k-instruct-FP8",
16 ]
17
18
19 @pytest.mark.skipif(not is_quant_method_supported("fp8"),
20 reason="FP8 is not supported on this GPU type.")
20 -@pytest.mark.parametrize("model", MODELS)
21 -def test_model_load_and_run(vllm_runner, model: str):
22 - with vllm_runner(model) as llm:
21 +@pytest.mark.parametrize("model_id", MODELS)
22 +def test_model_load_and_run(vllm_runner, model_id: str):
23 + with vllm_runner(model_id) as llm:
24 + # note: this does not test accuracy, just that we can run through
25 + # see lm-eval tests for accuracy
26 + outputs = llm.generate_greedy(prompts=["Hello my name is"],
27 + max_tokens=10)
28 + print(outputs[0][1])
29 +
30 +
31 +KV_CACHE_MODELS = [
32 + # Deprecated AutoFP8 format using .kv_scale
33 + "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
34 + # AutoFP8 format using separate .k_scale and .v_scale
35 + "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",
36 +]
37 +
38 +
39 +@pytest.mark.skipif(not is_quant_method_supported("fp8"),
40 + reason="FP8 is not supported on this GPU type.")
41 +@pytest.mark.parametrize("model_id", KV_CACHE_MODELS)
42 +def test_kv_cache_model_load_and_run(vllm_runner, model_id: str):
43 + with vllm_runner(model_id, kv_cache_dtype="fp8") as llm:
44 +
45 + model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
46 + attn = model.model.layers[0].self_attn.attn
47 + assert isinstance(attn.quant_method, Fp8KVCacheMethod)
48 + # NOTE: it is valid for scales to be 1.0 (default value), but we know
49 + # these checkpoints have scales < 1.0
50 + assert 0.0 < attn._k_scale < 1.0
51 + assert 0.0 < attn._v_scale < 1.0
52 +
53 # note: this does not test accuracy, just that we can run through
54 # see lm-eval tests for accuracy
55 outputs = llm.generate_greedy(prompts=["Hello my name is"],
vllm/_custom_ops.py
+11 -7
@@ -84,7 +84,8 @@ def paged_attention_v1(
84 max_seq_len: int,
85 alibi_slopes: Optional[torch.Tensor],
86 kv_cache_dtype: str,
87 - kv_scale: float,
87 + k_scale: float,
88 + v_scale: float,
89 tp_rank: int = 0,
90 blocksparse_local_blocks: int = 0,
91 blocksparse_vert_stride: int = 0,
@@ -94,8 +95,9 @@ def paged_attention_v1(
95 torch.ops._C.paged_attention_v1(
96 out, query, key_cache, value_cache, num_kv_heads, scale, block_tables,
97 seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
97 - kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride,
98 - blocksparse_block_size, blocksparse_head_sliding_step)
98 + k_scale, v_scale, tp_rank, blocksparse_local_blocks,
99 + blocksparse_vert_stride, blocksparse_block_size,
100 + blocksparse_head_sliding_step)
101
102
103 def paged_attention_v2(
@@ -114,7 +116,8 @@ def paged_attention_v2(
116 max_seq_len: int,
117 alibi_slopes: Optional[torch.Tensor],
118 kv_cache_dtype: str,
117 - kv_scale: float,
119 + k_scale: float,
120 + v_scale: float,
121 tp_rank: int = 0,
122 blocksparse_local_blocks: int = 0,
123 blocksparse_vert_stride: int = 0,
@@ -124,7 +127,7 @@ def paged_attention_v2(
127 torch.ops._C.paged_attention_v2(
128 out, exp_sum, max_logits, tmp_out, query, key_cache, value_cache,
129 num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len,
127 - alibi_slopes, kv_cache_dtype, kv_scale, tp_rank,
130 + alibi_slopes, kv_cache_dtype, k_scale, v_scale, tp_rank,
131 blocksparse_local_blocks, blocksparse_vert_stride,
132 blocksparse_block_size, blocksparse_head_sliding_step)
133
@@ -374,11 +377,12 @@ def reshape_and_cache(
377 value_cache: torch.Tensor,
378 slot_mapping: torch.Tensor,
379 kv_cache_dtype: str,
377 - kv_scale: float,
380 + k_scale: float,
381 + v_scale: float,
382 ) -> None:
383 torch.ops._C_cache_ops.reshape_and_cache(key, value, key_cache,
384 value_cache, slot_mapping,
381 - kv_cache_dtype, kv_scale)
385 + kv_cache_dtype, k_scale, v_scale)
386
387
388 def reshape_and_cache_flash(
vllm/_ipex_ops.py
+6 -3
@@ -59,7 +59,8 @@ class ipex_ops:
59 max_context_len: int,
60 alibi_slopes: Optional[torch.Tensor],
61 kv_cache_dtype: str,
62 - kv_scale: float,
62 + k_scale: float,
63 + v_scale: float,
64 tp_rank: int = 0,
65 blocksparse_local_blocks: int = 0,
66 blocksparse_vert_stride: int = 0,
@@ -99,7 +100,8 @@ class ipex_ops:
100 max_context_len: int,
101 alibi_slopes: Optional[torch.Tensor],
102 kv_cache_dtype: str,
102 - kv_scale: float,
103 + k_scale: float,
104 + v_scale: float,
105 tp_rank: int = 0,
106 blocksparse_local_blocks: int = 0,
107 blocksparse_vert_stride: int = 0,
@@ -227,7 +229,8 @@ class ipex_ops:
229 value_cache: torch.Tensor,
230 slot_mapping: torch.Tensor,
231 kv_cache_dtype: str,
230 - kv_scale: float,
232 + k_scale: float,
233 + v_scale: float,
234 ) -> None:
235 assert kv_cache_dtype == "auto"
236 ipex.llm.modules.PagedAttention.reshape_and_cache(
vllm/attention/backends/abstract.py
+2 -1
@@ -134,7 +134,8 @@ class AttentionImpl(ABC, Generic[T]):
134 value: torch.Tensor,
135 kv_cache: torch.Tensor,
136 attn_metadata: T,
137 - kv_scale: float = 1.0,
137 + k_scale: float = 1.0,
138 + v_scale: float = 1.0,
139 attn_type: AttentionType = AttentionType.DECODER,
140 ) -> torch.Tensor:
141 raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+6 -3
@@ -327,7 +327,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
327 value: torch.Tensor,
328 kv_cache: torch.Tensor,
329 attn_metadata: BlocksparseFlashAttentionMetadata,
330 - kv_scale: float = 1.0,
330 + k_scale: float = 1.0,
331 + v_scale: float = 1.0,
332 attn_type: AttentionType = AttentionType.DECODER,
333 ) -> torch.Tensor:
334 """Forward pass with FlashAttention and PagedAttention.
@@ -368,7 +369,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
369 value_cache,
370 attn_metadata.slot_mapping,
371 self.kv_cache_dtype,
371 - kv_scale,
372 + k_scale,
373 + v_scale,
374 )
375
376 if prefill_meta := attn_metadata.prefill_metadata:
@@ -405,7 +407,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
407 self.num_kv_heads,
408 self.scale,
409 self.alibi_slopes,
408 - kv_scale,
410 + k_scale,
411 + v_scale,
412 tp_rank=self.tp_rank,
413 blocksparse_local_blocks=self.local_blocks,
414 blocksparse_vert_stride=self.vert_stride,
vllm/attention/backends/flash_attn.py
+4 -2
@@ -256,7 +256,8 @@ class FlashAttentionImpl(AttentionImpl):
256 value: torch.Tensor,
257 kv_cache: torch.Tensor,
258 attn_metadata: FlashAttentionMetadata,
259 - kv_scale: float = 1.0,
259 + k_scale: float = 1.0,
260 + v_scale: float = 1.0,
261 attn_type: AttentionType = AttentionType.DECODER,
262 ) -> torch.Tensor:
263 """Forward pass with FlashAttention.
@@ -277,7 +278,8 @@ class FlashAttentionImpl(AttentionImpl):
278 "FlashAttentionImpl")
279
280 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
280 - assert kv_scale == 1.0, "kv_scale is not supported in FlashAttention."
281 + assert k_scale == 1.0 and v_scale == 1.0, (
282 + "key/v_scale is not supported in FlashAttention.")
283
284 num_tokens, hidden_size = query.shape
285 # Reshape the query, key, and value tensors.
vllm/attention/backends/flashinfer.py
+4 -2
@@ -223,10 +223,12 @@ class FlashInferImpl(AttentionImpl):
223 value: torch.Tensor,
224 kv_cache: Optional[torch.Tensor],
225 attn_metadata: FlashInferMetadata,
226 - kv_scale: float = 1.0,
226 + k_scale: float = 1.0,
227 + v_scale: float = 1.0,
228 attn_type: AttentionType = AttentionType.DECODER,
229 ) -> torch.Tensor:
229 - assert kv_scale == 1.0
230 + assert k_scale == 1.0 and v_scale == 1.0, (
231 + "key/v_scale is not supported in FlashInfer.")
232 if attn_type != AttentionType.DECODER:
233 raise NotImplementedError("Encoder self-attention and "
234 "encoder/decoder cross-attention "
vllm/attention/backends/ipex_attn.py
+9 -5
@@ -156,7 +156,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
156 value: torch.Tensor,
157 kv_cache: Optional[torch.Tensor],
158 attn_metadata: IpexAttnMetadata, # type: ignore
159 - kv_scale: float = 1.0,
159 + k_scale: float = 1.0,
160 + v_scale: float = 1.0,
161 attn_type: AttentionType = AttentionType.DECODER,
162 ) -> torch.Tensor:
163 """Forward pass with IPEX varlen_attention and PagedAttention.
@@ -170,7 +171,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
171 Returns:
172 shape = [num_tokens, num_heads * head_size]
173 """
173 - assert kv_scale == 1.0
174 + assert k_scale == 1.0 and v_scale == 1.0
175 if attn_type != AttentionType.DECODER:
176 raise NotImplementedError("Encoder self-attention and "
177 "encoder/decoder cross-attention "
@@ -192,7 +193,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
193 value_cache,
194 attn_metadata.slot_mapping.flatten(),
195 self.kv_cache_dtype,
195 - kv_scale,
196 + k_scale,
197 + v_scale,
198 )
199
200 if attn_metadata.is_prompt:
@@ -273,7 +275,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
275 max_seq_len,
276 self.alibi_slopes,
277 self.kv_cache_dtype,
276 - kv_scale,
278 + k_scale,
279 + v_scale,
280 )
281 else:
282 # Run PagedAttention V2.
@@ -305,7 +308,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
308 max_seq_len,
309 self.alibi_slopes,
310 self.kv_cache_dtype,
308 - kv_scale,
311 + k_scale,
312 + v_scale,
313 )
314
315 # Reshape the output tensor.
vllm/attention/backends/pallas.py
+3 -2
@@ -131,7 +131,8 @@ class PallasAttentionBackendImpl(AttentionImpl):
131 value: torch.Tensor,
132 kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
133 attn_metadata: PallasMetadata,
134 - kv_scale: float = 1.0,
134 + k_scale: float = 1.0,
135 + v_scale: float = 1.0,
136 attn_type: AttentionType = AttentionType.DECODER,
137 ) -> torch.Tensor:
138 """Forward pass with Pallas attention.
@@ -146,7 +147,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
147 Returns:
148 shape = [batch_size, seq_len, num_heads * head_size]
149 """
149 - assert kv_scale == 1.0
150 + assert k_scale == 1.0 and v_scale == 1.0
151 if attn_type != AttentionType.DECODER:
152 raise NotImplementedError("Encoder self-attention and "
153 "encoder/decoder cross-attention "
vllm/attention/backends/rocm_flash_attn.py
+6 -3
@@ -296,7 +296,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
296 value: torch.Tensor,
297 kv_cache: torch.Tensor,
298 attn_metadata: ROCmFlashAttentionMetadata,
299 - kv_scale: float = 1.0,
299 + k_scale: float = 1.0,
300 + v_scale: float = 1.0,
301 attn_type: AttentionType = AttentionType.DECODER,
302 ) -> torch.Tensor:
303 """Forward pass with FlashAttention and PagedAttention.
@@ -336,7 +337,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
337 value_cache,
338 attn_metadata.slot_mapping,
339 self.kv_cache_dtype,
339 - kv_scale,
340 + k_scale,
341 + v_scale,
342 )
343
344 num_prefill_tokens = attn_metadata.num_prefill_tokens
@@ -456,7 +458,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
458 self.num_kv_heads,
459 self.scale,
460 self.alibi_slopes,
459 - kv_scale,
461 + k_scale,
462 + v_scale,
463 )
464
465 # Reshape the output tensor.
vllm/attention/backends/torch_sdpa.py
+7 -4
@@ -144,7 +144,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
144 value: torch.Tensor,
145 kv_cache: Optional[torch.Tensor],
146 attn_metadata: TorchSDPAMetadata, # type: ignore
147 - kv_scale: float = 1.0,
147 + k_scale: float = 1.0,
148 + v_scale: float = 1.0,
149 attn_type: AttentionType = AttentionType.DECODER,
150 ) -> torch.Tensor:
151 """Forward pass with torch SDPA and PagedAttention.
@@ -158,7 +159,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
159 Returns:
160 shape = [num_tokens, num_heads * head_size]
161 """
161 - assert kv_scale == 1.0
162 + assert k_scale == 1.0 and v_scale == 1.0
163 if attn_type != AttentionType.DECODER:
164 raise NotImplementedError("Encoder self-attention and "
165 "encoder/decoder cross-attention "
@@ -176,7 +177,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
177 PagedAttention.write_to_paged_cache(key, value, key_cache,
178 value_cache,
179 attn_metadata.slot_mapping,
179 - self.kv_cache_dtype, kv_scale)
180 + self.kv_cache_dtype, k_scale,
181 + v_scale)
182
183 if attn_metadata.is_prompt:
184 assert attn_metadata.seq_lens is not None
@@ -239,7 +241,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
241 self.num_kv_heads,
242 self.scale,
243 self.alibi_slopes,
242 - kv_scale,
244 + k_scale,
245 + v_scale,
246 )
247
248 # Reshape the output tensor.
vllm/attention/backends/xformers.py
+5 -3
@@ -427,7 +427,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
427 value: Optional[torch.Tensor],
428 kv_cache: Optional[torch.Tensor],
429 attn_metadata: "XFormersMetadata",
430 - kv_scale: float = 1.0,
430 + k_scale: float = 1.0,
431 + v_scale: float = 1.0,
432 attn_type: AttentionType = AttentionType.DECODER,
433 ) -> torch.Tensor:
434 """Forward pass with xFormers and PagedAttention.
@@ -531,7 +532,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
532 value_cache,
533 updated_slot_mapping,
534 self.kv_cache_dtype,
534 - kv_scale)
535 + k_scale, v_scale)
536
537 if attn_type != AttentionType.ENCODER:
538 # Decoder self-attention supports chunked prefill.
@@ -620,7 +621,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
621 self.num_kv_heads,
622 self.scale,
623 self.alibi_slopes,
623 - kv_scale,
624 + k_scale,
625 + v_scale,
626 )
627
628 # Reshape the output tensor.
vllm/attention/layer.py
+8 -6
@@ -47,13 +47,14 @@ class Attention(nn.Module):
47 if num_kv_heads is None:
48 num_kv_heads = num_heads
49
50 - # The default kv_scale is set to 1.0. This is ignored
50 + # The default k/v_scale is set to 1.0. This is ignored
51 # when kv-cache is not fp8, and should be used with
52 # kv-cache in fp8_e5m2. For kv-cache in fp8_e4m3, we
53 - # expect the pre-quantized kv_scale to be loaded along
53 + # expect the pre-quantized k/v_scale to be loaded along
54 # with the model weights.
55 self.kv_cache_dtype = kv_cache_dtype
56 - self._kv_scale = 1.0
56 + self._k_scale = 1.0
57 + self._v_scale = 1.0
58 quant_method = quant_config.get_quant_method(
59 self) if quant_config else None
60 if quant_method is not None:
@@ -66,8 +67,8 @@ class Attention(nn.Module):
67 "fp8 checkpoints.")
68 # When FP8 quantization is enabled, we make a parameter
69 # "kv_scale" so that it can be loaded from FP8 checkpoint.
69 - # The kv_scale will then be converted back to self._kv_scale
70 - # in a native float32 value after weight loading.
70 + # The k/v_scale will then be converted back to
71 + # self._kv_scale in a native float32 value after weight loading
72 self.quant_method = quant_method
73 self.quant_method.create_weights(self)
74
@@ -98,7 +99,8 @@ class Attention(nn.Module):
99 value,
100 kv_cache,
101 attn_metadata,
101 - self._kv_scale,
102 + self._k_scale,
103 + self._v_scale,
104 attn_type=attn_type)
105
106 def extra_repr(self) -> str:
vllm/attention/ops/ipex_attn.py
+4 -2
@@ -45,7 +45,8 @@ class PagedAttention:
45 value_cache: torch.Tensor,
46 slot_mapping: torch.Tensor,
47 kv_cache_dtype: str,
48 - kv_scale: float,
48 + k_scale: float,
49 + v_scale: float,
50 *args,
51 ) -> None:
52 ipex_modules.PagedAttention.reshape_and_cache(
@@ -64,7 +65,8 @@ class PagedAttention:
65 num_kv_heads: int,
66 scale: float,
67 alibi_slopes: Optional[torch.Tensor],
67 - kv_scale: float,
68 + k_scale: float,
69 + v_scale: float,
70 *args,
71 ) -> torch.Tensor:
72 output = torch.empty_like(query)
vllm/attention/ops/paged_attn.py
+10 -5
@@ -66,7 +66,8 @@ class PagedAttention:
66 value_cache: torch.Tensor,
67 slot_mapping: torch.Tensor,
68 kv_cache_dtype: str,
69 - kv_scale: float,
69 + k_scale: float,
70 + v_scale: float,
71 ) -> None:
72 ops.reshape_and_cache(
73 key,
@@ -75,7 +76,8 @@ class PagedAttention:
76 value_cache,
77 slot_mapping.flatten(),
78 kv_cache_dtype,
78 - kv_scale,
79 + k_scale,
80 + v_scale,
81 )
82
83 @staticmethod
@@ -90,7 +92,8 @@ class PagedAttention:
92 num_kv_heads: int,
93 scale: float,
94 alibi_slopes: Optional[torch.Tensor],
93 - kv_scale: float,
95 + k_scale: float,
96 + v_scale: float,
97 tp_rank: int = 0,
98 blocksparse_local_blocks: int = 0,
99 blocksparse_vert_stride: int = 0,
@@ -135,7 +138,8 @@ class PagedAttention:
138 max_seq_len,
139 alibi_slopes,
140 kv_cache_dtype,
138 - kv_scale,
141 + k_scale,
142 + v_scale,
143 tp_rank,
144 blocksparse_local_blocks,
145 blocksparse_vert_stride,
@@ -172,7 +176,8 @@ class PagedAttention:
176 max_seq_len,
177 alibi_slopes,
178 kv_cache_dtype,
175 - kv_scale,
179 + k_scale,
180 + v_scale,
181 tp_rank,
182 blocksparse_local_blocks,
183 blocksparse_vert_stride,
vllm/model_executor/layers/linear.py
+9
@@ -196,6 +196,15 @@ class ReplicatedLinear(LinearBase):
196 else:
197 self.register_parameter("bias", None)
198
199 + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
200 + # If the weight on disk does not have a shape, give it one
201 + # (such scales for AutoFp8).
202 + if len(loaded_weight.shape) == 0:
203 + loaded_weight = loaded_weight.reshape(1)
204 +
205 + assert param.size() == loaded_weight.size()
206 + param.data.copy_(loaded_weight)
207 +
208 def forward(self, x: torch.Tensor) -> torch.Tensor:
209 bias = self.bias if not self.skip_bias_add else None
210 assert self.quant_method is not None
vllm/model_executor/layers/quantization/fp8.py
+38 -13
@@ -407,31 +407,56 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
407 self.quant_config = quant_config
408
409 def create_weights(self, layer: torch.nn.Module):
410 - """Create "weight" (aka kv_scale) for an attention layer.
410 + """Create "weight" (aka k_scale and v_scale) for an attention layer.
411
412 Args:
413 layer: The layer that is using the QuantizeMethodBase factory.
414 """
415 - # Initialize the KV cache scale to 1.0 as the default value.
416 - # If the kv_scale appears in the checkpoint, it will be
415 + # Initialize the KV cache scales to -1.0, which is an invalid value.
416 + # If the k/v_scale appears in the checkpoint, it will be
417 # overwritten when loading weights.
418 - layer.kv_scale = Parameter(torch.tensor(1.0), requires_grad=False)
418 + layer.k_scale = Parameter(torch.tensor(-1.0), requires_grad=False)
419 + layer.v_scale = Parameter(torch.tensor(-1.0), requires_grad=False)
420
421 def apply(self, layer: torch.nn.Module) -> torch.Tensor:
422 raise RuntimeError("Fp8KVCacheMethod.apply should not be called.")
423
424 def process_weights_after_loading(self, layer: Module) -> None:
424 - # If the kv-cache dtype is auto, we enforce the kv-scale to be 1.0
425 + # If the kv-cache dtype is auto, we enforce the k/v_scale to be 1.0
426 # regardless whether the kv-scale is available in the checkpoint.
427 if layer.kv_cache_dtype != "auto":
427 - kv_scale = layer.kv_scale.to("cpu").tolist()
428 - if not isinstance(kv_scale, float):
428 + if layer.k_scale > 0.0 and layer.v_scale > 0.0:
429 + # We prefer to use separate k_scale and v_scale if present
430 + k_scale = layer.k_scale.to("cpu").tolist()
431 + v_scale = layer.v_scale.to("cpu").tolist()
432 + elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
433 + # If no scales were loaded (both scales are invalid negative
434 + # values), use the default value of 1.0
435 + k_scale = Parameter(torch.tensor(1.0), requires_grad=False)
436 + v_scale = Parameter(torch.tensor(1.0), requires_grad=False)
437 + else:
438 + # If we find a single kv_scale in the checkpoint, we remap
439 + # kv_scale to k_scale during weight loading, and duplicate
440 + # k_scale to v_scale here
441 + assert layer.k_scale > 0.0
442 + scale_to_duplicate = max(layer.k_scale, layer.v_scale)
443 + k_scale = scale_to_duplicate.to("cpu").tolist()
444 + v_scale = scale_to_duplicate.to("cpu").tolist()
445 +
446 + if not isinstance(k_scale, float) or not isinstance(
447 + v_scale, float):
448 raise ValueError("Only support per-tensor scaling factor "
449 "for fp8 KV cache")
431 - layer._kv_scale = kv_scale
432 - if layer._kv_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
450 +
451 + # These are used in the final Attention.forward()
452 + layer._k_scale = k_scale
453 + layer._v_scale = v_scale
454 + if (layer._k_scale == 1.0 and layer._v_scale == 1.0
455 + and "e5m2" not in layer.kv_cache_dtype):
456 print_warning_once(
434 - "Using KV cache scaling factor 1.0 for fp8_e4m3. This may "
435 - "cause accuracy issues. Please make sure kv-cache scaling "
436 - "factor is available in the fp8 checkpoint.")
437 - del layer.kv_scale
457 + "Using KV cache scaling factor 1.0 for fp8_e4m3. This "
458 + "may cause accuracy issues. Please make sure k/v_scale "
459 + "scaling factors are available in the fp8 checkpoint.")
460 +
461 + del layer.k_scale
462 + del layer.v_scale
vllm/model_executor/model_loader/weight_utils.py
+53 -5
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
22 from vllm.model_executor.layers.quantization import (QuantizationConfig,
23 get_quantization_config)
24 from vllm.model_executor.layers.quantization.schema import QuantParamSchema
25 +from vllm.utils import print_warning_once
26
27 logger = init_logger(__name__)
28
@@ -431,11 +432,6 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
432 def default_weight_loader(param: torch.Tensor,
433 loaded_weight: torch.Tensor) -> None:
434 """Default weight loader."""
434 - # If the weight on disk does not have a shape, give it one
435 - # (such scales for AutoFp8).
436 - if len(loaded_weight.shape) == 0:
437 - loaded_weight = loaded_weight.reshape(1)
438 -
435 assert param.size() == loaded_weight.size()
436 param.data.copy_(loaded_weight)
437
@@ -462,3 +458,55 @@ def initialize_dummy_weights(
458 param.data.copy_(tmp_param)
459 else:
460 param.uniform_(low, high)
461 +
462 +
463 +def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
464 + """Remap the name of FP8 k/v_scale parameters.
465 +
466 + This function handles the remapping of FP8 k/v_scale parameter names.
467 + It detects if the given name ends with a suffix and attempts to remap
468 + it to the expected name format in the model. If the remapped name is not
469 + found in the params_dict, a warning is printed and None is returned.
470 +
471 + Args:
472 + name (str): The original loaded checkpoint parameter name.
473 + params_dict (dict): Dictionary containing the model's named parameters.
474 +
475 + Returns:
476 + str: The remapped parameter name if successful, or the original name
477 + if no remapping is needed.
478 + None: If the remapped name is not found in params_dict.
479 + """
480 + if name.endswith(".kv_scale"):
481 + print_warning_once(
482 + "DEPRECATED. Found kv_scale in the checkpoint. "
483 + "This format is deprecated in favor of separate k_scale and "
484 + "v_scale tensors and will be removed in a future release. "
485 + "Functionally, we will remap kv_scale to k_scale and duplicate "
486 + "k_scale to v_scale")
487 + # NOTE: we remap the deprecated kv_scale to k_scale
488 + remapped_name = name.replace(".kv_scale", ".attn.k_scale")
489 + if remapped_name not in params_dict:
490 + print_warning_once(
491 + f"Found kv_scale in the checkpoint (e.g. {name}), "
492 + "but not found the expected name in the model "
493 + f"(e.g. {remapped_name}). kv_scale is "
494 + "not loaded.")
495 + return None
496 + return remapped_name
497 +
498 + possible_scale_names = [".k_scale", ".v_scale"]
499 + for scale_name in possible_scale_names:
500 + if name.endswith(scale_name):
501 + remapped_name = name.replace(scale_name, f".attn{scale_name}")
502 + if remapped_name not in params_dict:
503 + print_warning_once(
504 + f"Found {scale_name} in the checkpoint (e.g. {name}), "
505 + "but not found the expected name in the model "
506 + f"(e.g. {remapped_name}). {scale_name} is "
507 + "not loaded.")
508 + return None
509 + return remapped_name
510 +
511 + # If there were no matches, return the untouched param name
512 + return name
vllm/model_executor/models/llama.py
+5 -14
@@ -44,10 +44,10 @@ from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
46 from vllm.model_executor.model_loader.weight_utils import (
47 - default_weight_loader, kv_cache_scales_loader)
47 + default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
48 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.sequence import IntermediateTensors, SamplerOutput
50 -from vllm.utils import is_hip, print_warning_once
50 +from vllm.utils import is_hip
51
52 from .interfaces import SupportsLoRA
53 from .utils import is_pp_missing_parameter, make_layers
@@ -460,18 +460,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
460 if name.endswith(".bias") and name not in params_dict:
461 continue
462 # Remapping the name of FP8 kv-scale.
463 - if name.endswith("kv_scale"):
464 - remapped_kv_scale_name = name.replace(
465 - ".kv_scale", ".attn.kv_scale")
466 - if remapped_kv_scale_name not in params_dict:
467 - print_warning_once(
468 - f"Found kv scale in the checkpoint (e.g. {name}), "
469 - "but not found the expected name in the model "
470 - f"(e.g. {remapped_kv_scale_name}). kv-scale is "
471 - "not loaded.")
472 - continue
473 - else:
474 - name = remapped_kv_scale_name
463 + name = maybe_remap_kv_scale_name(name, params_dict)
464 + if name is None:
465 + continue
466
467 if is_pp_missing_parameter(name, self):
468 continue
vllm/model_executor/models/mixtral.py
+6 -15
@@ -42,10 +42,10 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 +from vllm.model_executor.model_loader.weight_utils import (
46 + default_weight_loader, maybe_remap_kv_scale_name)
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.sequence import IntermediateTensors, SamplerOutput
48 -from vllm.utils import print_warning_once
49
50 from .interfaces import SupportsLoRA
51
@@ -415,19 +415,10 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
415 if name.endswith(".bias") and name not in params_dict:
416 continue
417 # Remapping the name of FP8 kv-scale.
418 - if name.endswith("kv_scale"):
419 - remapped_kv_scale_name = name.replace(
420 - ".kv_scale", ".attn.kv_scale")
421 - if remapped_kv_scale_name not in params_dict:
422 - print_warning_once(
423 - "Found kv scale in the checkpoint "
424 - f"(e.g. {name}), but not found the expected "
425 - f"name in the model "
426 - f"(e.g. {remapped_kv_scale_name}). "
427 - "kv-scale is not loaded.")
428 - continue
429 - else:
430 - name = remapped_kv_scale_name
418 + name = maybe_remap_kv_scale_name(name, params_dict)
419 + if name is None:
420 + continue
421 +
422 param = params_dict[name]
423 weight_loader = getattr(param, "weight_loader",
424 default_weight_loader)
vllm/model_executor/models/qwen2.py
+6 -14
@@ -43,10 +43,10 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 ParallelLMHead, VocabParallelEmbedding)
46 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46 +from vllm.model_executor.model_loader.weight_utils import (
47 + default_weight_loader, maybe_remap_kv_scale_name)
48 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.sequence import IntermediateTensors, SamplerOutput
49 -from vllm.utils import print_warning_once
50
51 from .interfaces import SupportsLoRA
52
@@ -382,18 +382,10 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
382 if name.endswith(".bias") and name not in params_dict:
383 continue
384 # Remapping the name of FP8 kv-scale.
385 - if name.endswith("kv_scale"):
386 - remapped_kv_scale_name = name.replace(
387 - ".kv_scale", ".attn.kv_scale")
388 - if remapped_kv_scale_name not in params_dict:
389 - print_warning_once(
390 - f"Found kv scale in the checkpoint (e.g. {name}), "
391 - "but not found the expected name in the model "
392 - f"(e.g. {remapped_kv_scale_name}). kv-scale is "
393 - "not loaded.")
394 - continue
395 - else:
396 - name = remapped_kv_scale_name
385 + name = maybe_remap_kv_scale_name(name, params_dict)
386 + if name is None:
387 + continue
388 +
389 param = params_dict[name]
390 weight_loader = getattr(param, "weight_loader",
391 default_weight_loader)