A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Kernel][Attention] Separate `Attention.kv_scale` into `k_scale` and `v_scale` (#6081)
Michael Goin committed
Jul 16, 2024 at 18:31 UTC
978aed53004b82877bd2af0f10afff1826d7194d
33 files changed
+317
-185
benchmarks/kernels/benchmark_paged_attention.py
+5
-3
@@ -100,7 +100,7 @@ def main(
100
start_time = time.perf_counter()
101
102
# Using default kv_scale
103
- kv_scale = 1.0
103
+ k_scale = v_scale = 1.0
104
105
for _ in range(num_iters):
106
if version == "v1":
@@ -117,7 +117,8 @@ def main(
117
max_seq_len,
118
alibi_slopes,
119
kv_cache_dtype,
120
- kv_scale,
120
+ k_scale,
121
+ v_scale,
122
)
123
elif version == "v2":
124
ops.paged_attention_v2(
@@ -136,7 +137,8 @@ def main(
137
max_seq_len,
138
alibi_slopes,
139
kv_cache_dtype,
139
- kv_scale,
140
+ k_scale,
141
+ v_scale,
142
)
143
else:
144
raise ValueError(f"Invalid version: {version}")
csrc/attention/attention_kernels.cu
+27
-26
@@ -105,9 +105,9 @@ __device__ void paged_attention_kernel(
105
const int max_num_blocks_per_seq,
106
const float* __restrict__ alibi_slopes, // [num_heads]
107
const int q_stride, const int kv_block_stride, const int kv_head_stride,
108
- const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
109
- const int blocksparse_vert_stride, const int blocksparse_block_size,
110
- const int blocksparse_head_sliding_step) {
108
+ const float k_scale, const float v_scale, const int tp_rank,
109
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
110
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
111
const int seq_idx = blockIdx.y;
112
const int partition_idx = blockIdx.z;
113
const int max_num_partitions = gridDim.z;
@@ -285,7 +285,7 @@ __device__ void paged_attention_kernel(
285
Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
286
k_ptr + offset1 * BLOCK_SIZE * x + offset2);
287
k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(
288
- k_vec_quant, kv_scale);
288
+ k_vec_quant, k_scale);
289
}
290
}
291
@@ -415,7 +415,7 @@ __device__ void paged_attention_kernel(
415
*reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
416
// Vector conversion from V_quant_vec to V_vec.
417
v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec,
418
- kv_scale);
418
+ v_scale);
419
}
420
if (block_idx == num_seq_blocks - 1) {
421
// NOTE(woosuk): When v_vec contains the tokens that are out of the
@@ -513,15 +513,15 @@ __global__ void paged_attention_v1_kernel(
513
const int max_num_blocks_per_seq,
514
const float* __restrict__ alibi_slopes, // [num_heads]
515
const int q_stride, const int kv_block_stride, const int kv_head_stride,
516
- const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
517
- const int blocksparse_vert_stride, const int blocksparse_block_size,
518
- const int blocksparse_head_sliding_step) {
516
+ const float k_scale, const float v_scale, const int tp_rank,
517
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
518
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
519
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
520
KV_DTYPE, IS_BLOCK_SPARSE>(
521
/* exp_sums */ nullptr, /* max_logits */ nullptr, out, q, k_cache,
522
v_cache, num_kv_heads, scale, block_tables, seq_lens,
523
max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride,
524
- kv_head_stride, kv_scale, tp_rank, blocksparse_local_blocks,
524
+ kv_head_stride, k_scale, v_scale, tp_rank, blocksparse_local_blocks,
525
blocksparse_vert_stride, blocksparse_block_size,
526
blocksparse_head_sliding_step);
527
}
@@ -549,14 +549,14 @@ __global__ void paged_attention_v2_kernel(
549
const int max_num_blocks_per_seq,
550
const float* __restrict__ alibi_slopes, // [num_heads]
551
const int q_stride, const int kv_block_stride, const int kv_head_stride,
552
- const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
553
- const int blocksparse_vert_stride, const int blocksparse_block_size,
554
- const int blocksparse_head_sliding_step) {
552
+ const float k_scale, const float v_scale, const int tp_rank,
553
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
554
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
555
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
556
KV_DTYPE, IS_BLOCK_SPARSE, PARTITION_SIZE>(
557
exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
558
block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes, q_stride,
559
- kv_block_stride, kv_head_stride, kv_scale, tp_rank,
559
+ kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank,
560
blocksparse_local_blocks, blocksparse_vert_stride, blocksparse_block_size,
561
blocksparse_head_sliding_step);
562
}
@@ -682,7 +682,7 @@ __global__ void paged_attention_v2_reduce_kernel(
682
out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
683
scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
684
alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
685
- kv_scale, tp_rank, blocksparse_local_blocks, \
685
+ k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
686
blocksparse_vert_stride, blocksparse_block_size, \
687
blocksparse_head_sliding_step);
688
@@ -694,8 +694,8 @@ void paged_attention_v1_launcher(
694
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
695
torch::Tensor& value_cache, int num_kv_heads, float scale,
696
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
697
- const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
698
- const int tp_rank, const int blocksparse_local_blocks,
697
+ const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
698
+ float v_scale, const int tp_rank, const int blocksparse_local_blocks,
699
const int blocksparse_vert_stride, const int blocksparse_block_size,
700
const int blocksparse_head_sliding_step) {
701
int num_seqs = query.size(0);
@@ -770,7 +770,7 @@ void paged_attention_v1_launcher(
770
paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
771
IS_BLOCK_SPARSE>( \
772
out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
773
- seq_lens, max_seq_len, alibi_slopes, kv_scale, tp_rank, \
773
+ seq_lens, max_seq_len, alibi_slopes, k_scale, v_scale, tp_rank, \
774
blocksparse_local_blocks, blocksparse_vert_stride, \
775
blocksparse_block_size, blocksparse_head_sliding_step);
776
@@ -815,8 +815,8 @@ void paged_attention_v1(
815
torch::Tensor& seq_lens, // [num_seqs]
816
int64_t block_size, int64_t max_seq_len,
817
const c10::optional<torch::Tensor>& alibi_slopes,
818
- const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
819
- const int64_t blocksparse_local_blocks,
818
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
819
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
820
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
821
const int64_t blocksparse_head_sliding_step) {
822
const bool is_block_sparse = (blocksparse_vert_stride > 1);
@@ -833,7 +833,7 @@ void paged_attention_v1(
833
exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
834
value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
835
seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
836
- kv_block_stride, kv_head_stride, kv_scale, tp_rank, \
836
+ kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
837
blocksparse_local_blocks, blocksparse_vert_stride, \
838
blocksparse_block_size, blocksparse_head_sliding_step); \
839
vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
@@ -850,8 +850,8 @@ void paged_attention_v2_launcher(
850
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
851
torch::Tensor& value_cache, int num_kv_heads, float scale,
852
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
853
- const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
854
- const int tp_rank, const int blocksparse_local_blocks,
853
+ const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
854
+ float v_scale, const int tp_rank, const int blocksparse_local_blocks,
855
const int blocksparse_vert_stride, const int blocksparse_block_size,
856
const int blocksparse_head_sliding_step) {
857
int num_seqs = query.size(0);
@@ -932,8 +932,9 @@ void paged_attention_v2_launcher(
932
IS_BLOCK_SPARSE>( \
933
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
934
num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
935
- kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride, \
936
- blocksparse_block_size, blocksparse_head_sliding_step);
935
+ k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
936
+ blocksparse_vert_stride, blocksparse_block_size, \
937
+ blocksparse_head_sliding_step);
938
939
#define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
940
switch (is_block_sparse) { \
@@ -980,8 +981,8 @@ void paged_attention_v2(
981
torch::Tensor& seq_lens, // [num_seqs]
982
int64_t block_size, int64_t max_seq_len,
983
const c10::optional<torch::Tensor>& alibi_slopes,
983
- const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
984
- const int64_t blocksparse_local_blocks,
984
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
985
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
986
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
987
const int64_t blocksparse_head_sliding_step) {
988
const bool is_block_sparse = (blocksparse_vert_stride > 1);
csrc/cache.h
+2
-2
@@ -18,8 +18,8 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
18
void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
19
torch::Tensor& key_cache, torch::Tensor& value_cache,
20
torch::Tensor& slot_mapping,
21
- const std::string& kv_cache_dtype,
22
- const double kv_scale);
21
+ const std::string& kv_cache_dtype, const double k_scale,
22
+ const double v_scale);
23
24
void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
25
torch::Tensor& key_cache,
csrc/cache_kernels.cu
+11
-10
@@ -159,8 +159,8 @@ __global__ void reshape_and_cache_kernel(
159
// block_size]
160
const int64_t* __restrict__ slot_mapping, // [num_tokens]
161
const int key_stride, const int value_stride, const int num_heads,
162
- const int head_size, const int block_size, const int x,
163
- const float kv_scale) {
162
+ const int head_size, const int block_size, const int x, const float k_scale,
163
+ const float v_scale) {
164
const int64_t token_idx = blockIdx.x;
165
const int64_t slot_idx = slot_mapping[token_idx];
166
if (slot_idx < 0) {
@@ -196,9 +196,9 @@ __global__ void reshape_and_cache_kernel(
196
value_cache[tgt_value_idx] = tgt_value;
197
} else {
198
key_cache[tgt_key_idx] =
199
- fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, kv_scale);
199
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, k_scale);
200
value_cache[tgt_value_idx] =
201
- fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, kv_scale);
201
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, v_scale);
202
}
203
}
204
}
@@ -248,7 +248,7 @@ __global__ void reshape_and_cache_flash_kernel(
248
reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
249
reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
250
slot_mapping.data_ptr<int64_t>(), key_stride, value_stride, \
251
- num_heads, head_size, block_size, x, kv_scale);
251
+ num_heads, head_size, block_size, x, k_scale, v_scale);
252
253
void reshape_and_cache(
254
torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -258,7 +258,8 @@ void reshape_and_cache(
258
torch::Tensor&
259
value_cache, // [num_blocks, num_heads, head_size, block_size]
260
torch::Tensor& slot_mapping, // [num_tokens]
261
- const std::string& kv_cache_dtype, const double kv_scale) {
261
+ const std::string& kv_cache_dtype, const double k_scale,
262
+ const double v_scale) {
263
int num_tokens = key.size(0);
264
int num_heads = key.size(1);
265
int head_size = key.size(2);
@@ -318,13 +319,13 @@ namespace vllm {
319
template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
320
__global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
321
Tout* __restrict__ dst_cache,
321
- const float kv_scale,
322
+ const float scale,
323
const int64_t block_stride) {
324
const int64_t block_idx = blockIdx.x;
325
for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
326
int64_t idx = block_idx * block_stride + i;
327
dst_cache[idx] =
327
- fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], kv_scale);
328
+ fp8::scaled_convert<Tout, Tin, kv_dt>(src_cache[idx], scale);
329
}
330
}
331
@@ -333,11 +334,11 @@ __global__ void convert_fp8_kernel(const Tin* __restrict__ src_cache,
334
#define CALL_CONVERT_FP8(Tout, Tin, KV_DTYPE) \
335
vllm::convert_fp8_kernel<Tout, Tin, KV_DTYPE><<<grid, block, 0, stream>>>( \
336
reinterpret_cast<Tin*>(src_cache.data_ptr()), \
336
- reinterpret_cast<Tout*>(dst_cache.data_ptr()), kv_scale, block_stride);
337
+ reinterpret_cast<Tout*>(dst_cache.data_ptr()), scale, block_stride);
338
339
// Only for testing.
340
void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
340
- const double kv_scale, const std::string& kv_cache_dtype) {
341
+ const double scale, const std::string& kv_cache_dtype) {
342
torch::Device src_device = src_cache.device();
343
torch::Device dst_device = dst_cache.device();
344
TORCH_CHECK(src_device.is_cuda(), "src must be on a GPU")
csrc/cpu/attention.cpp
+6
-6
@@ -423,11 +423,11 @@ void paged_attention_v1(
423
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
424
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
425
int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
426
- const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
427
- const int64_t blocksparse_local_blocks,
426
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
427
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
428
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
429
const int64_t blocksparse_head_sliding_step) {
430
- TORCH_CHECK(kv_scale == 1.0f);
430
+ TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
431
TORCH_CHECK(blocksparse_vert_stride <= 1,
432
"CPU backend does not support blocksparse attention yet.");
433
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
@@ -742,11 +742,11 @@ void paged_attention_v2(
742
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
743
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
744
int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
745
- const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
746
- const int64_t blocksparse_local_blocks,
745
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
746
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
747
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
748
const int64_t blocksparse_head_sliding_step) {
749
- TORCH_CHECK(kv_scale == 1.0f);
749
+ TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
750
TORCH_CHECK(blocksparse_vert_stride <= 1,
751
"CPU backend does not support blocksparse attention yet.");
752
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
csrc/cpu/cache.cpp
+3
-2
@@ -107,8 +107,9 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
107
void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
108
torch::Tensor& key_cache, torch::Tensor& value_cache,
109
torch::Tensor& slot_mapping,
110
- const std::string& kv_cache_dtype, double kv_scale) {
111
- TORCH_CHECK(kv_scale == 1.0f);
110
+ const std::string& kv_cache_dtype, double k_scale,
111
+ double v_scale) {
112
+ TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
113
114
int num_tokens = key.size(0);
115
int num_heads = key.size(1);
csrc/cpu/torch_bindings.cpp
+5
-5
@@ -16,8 +16,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
16
" Tensor value_cache, int num_kv_heads, float scale,"
17
" Tensor block_tables, Tensor seq_lens, int block_size,"
18
" int max_seq_len, Tensor? alibi_slopes,"
19
- " str kv_cache_dtype, float kv_scale, int tp_rank,"
20
- " int blocksparse_local_blocks,"
19
+ " str kv_cache_dtype, float k_scale, float v_scale,"
20
+ " int tp_rank, int blocksparse_local_blocks,"
21
" int blocksparse_vert_stride, int blocksparse_block_size,"
22
" int blocksparse_head_sliding_step) -> ()");
23
ops.impl("paged_attention_v1", torch::kCPU, &paged_attention_v1);
@@ -30,8 +30,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
30
" Tensor value_cache, int num_kv_heads, float scale,"
31
" Tensor block_tables, Tensor seq_lens, int block_size,"
32
" int max_seq_len, Tensor? alibi_slopes,"
33
- " str kv_cache_dtype, float kv_scale, int tp_rank,"
34
- " int blocksparse_local_blocks,"
33
+ " str kv_cache_dtype, float k_scale, float v_scale,"
34
+ " int tp_rank, int blocksparse_local_blocks,"
35
" int blocksparse_vert_stride, int blocksparse_block_size,"
36
" int blocksparse_head_sliding_step) -> ()");
37
ops.impl("paged_attention_v2", torch::kCPU, &paged_attention_v2);
@@ -103,7 +103,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
103
" Tensor! key_cache, Tensor! value_cache,"
104
" Tensor slot_mapping,"
105
" str kv_cache_dtype,"
106
- " float kv_scale) -> ()");
106
+ " float k_scale, float v_scale) -> ()");
107
cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
108
}
109
csrc/ops.h
+4
-4
@@ -8,8 +8,8 @@ void paged_attention_v1(
8
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
9
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
10
int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
11
- const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
12
- const int64_t blocksparse_local_blocks,
11
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
12
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
13
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
14
const int64_t blocksparse_head_sliding_step);
15
@@ -19,8 +19,8 @@ void paged_attention_v2(
19
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
20
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
21
int64_t max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
22
- const std::string& kv_cache_dtype, double kv_scale, const int64_t tp_rank,
23
- const int64_t blocksparse_local_blocks,
22
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
23
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
24
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
25
const int64_t blocksparse_head_sliding_step);
26
csrc/torch_bindings.cpp
+5
-5
@@ -27,8 +27,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
27
" Tensor value_cache, int num_kv_heads, float scale,"
28
" Tensor block_tables, Tensor seq_lens, int block_size,"
29
" int max_seq_len, Tensor? alibi_slopes,"
30
- " str kv_cache_dtype, float kv_scale, int tp_rank,"
31
- " int blocksparse_local_blocks,"
30
+ " str kv_cache_dtype, float k_scale, float v_scale,"
31
+ " int tp_rank, int blocksparse_local_blocks,"
32
" int blocksparse_vert_stride, int blocksparse_block_size,"
33
" int blocksparse_head_sliding_step) -> ()");
34
ops.impl("paged_attention_v1", torch::kCUDA, &paged_attention_v1);
@@ -41,8 +41,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
41
" Tensor value_cache, int num_kv_heads, float scale,"
42
" Tensor block_tables, Tensor seq_lens, int block_size,"
43
" int max_seq_len, Tensor? alibi_slopes,"
44
- " str kv_cache_dtype, float kv_scale, int tp_rank,"
45
- " int blocksparse_local_blocks,"
44
+ " str kv_cache_dtype, float k_scale, float v_scale,"
45
+ " int tp_rank, int blocksparse_local_blocks,"
46
" int blocksparse_vert_stride, int blocksparse_block_size,"
47
" int blocksparse_head_sliding_step) -> ()");
48
ops.impl("paged_attention_v2", torch::kCUDA, &paged_attention_v2);
@@ -223,7 +223,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
223
" Tensor! key_cache, Tensor! value_cache,"
224
" Tensor slot_mapping,"
225
" str kv_cache_dtype,"
226
- " float kv_scale) -> ()");
226
+ " float k_scale, float v_scale) -> ()");
227
cache_ops.impl("reshape_and_cache", torch::kCUDA, &reshape_and_cache);
228
229
// Reshape the key and value tensors and cache them.
tests/kernels/test_attention.py
+5
-3
@@ -175,7 +175,7 @@ def test_paged_attention(
175
key_cache, value_cache = key_caches[0], value_caches[0]
176
177
# Using default kv_scale
178
- kv_scale = 1.0
178
+ k_scale = v_scale = 1.0
179
180
# Call the paged attention kernel.
181
output = torch.empty_like(query)
@@ -193,7 +193,8 @@ def test_paged_attention(
193
max_seq_len,
194
alibi_slopes,
195
kv_cache_dtype,
196
- kv_scale,
196
+ k_scale,
197
+ v_scale,
198
)
199
elif version == "v2":
200
num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
@@ -224,7 +225,8 @@ def test_paged_attention(
225
max_seq_len,
226
alibi_slopes,
227
kv_cache_dtype,
227
- kv_scale,
228
+ k_scale,
229
+ v_scale,
230
)
231
else:
232
raise AssertionError(f"Unknown version: {version}")
tests/kernels/test_blocksparse_attention.py
+5
-3
@@ -212,7 +212,7 @@ def test_paged_attention(
212
key_cache, value_cache = key_caches[0], value_caches[0]
213
214
# Using default kv_scale
215
- kv_scale = 1.0
215
+ k_scale = v_scale = 1.0
216
tp_rank = 0
217
218
# Call the paged attention kernel.
@@ -231,7 +231,8 @@ def test_paged_attention(
231
max_seq_len,
232
alibi_slopes,
233
kv_cache_dtype,
234
- kv_scale,
234
+ k_scale,
235
+ v_scale,
236
tp_rank=tp_rank,
237
blocksparse_local_blocks=blocksparse_local_blocks,
238
blocksparse_vert_stride=blocksparse_vert_stride,
@@ -267,7 +268,8 @@ def test_paged_attention(
268
max_seq_len,
269
alibi_slopes,
270
kv_cache_dtype,
270
- kv_scale,
271
+ k_scale,
272
+ v_scale,
273
tp_rank=tp_rank,
274
blocksparse_local_blocks=blocksparse_local_blocks,
275
blocksparse_vert_stride=blocksparse_vert_stride,
tests/kernels/test_cache.py
+2
-2
@@ -155,11 +155,11 @@ def test_reshape_and_cache(
155
cloned_value_cache = value_cache.clone()
156
157
# Using default kv_scale
158
- kv_scale = 1.0
158
+ k_scale = v_scale = 1.0
159
160
# Call the reshape_and_cache kernel.
161
ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping,
162
- kv_cache_dtype, kv_scale)
162
+ kv_cache_dtype, k_scale, v_scale)
163
164
if kv_cache_dtype == "fp8":
165
result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
tests/quantization/test_fp8.py
+35
-5
@@ -7,19 +7,49 @@ import torch
7
8
from tests.quantization.utils import is_quant_method_supported
9
from vllm import _custom_ops as ops
10
-from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
10
+from vllm.model_executor.layers.quantization.fp8 import (Fp8KVCacheMethod,
11
+ Fp8LinearMethod)
12
13
MODELS = [
13
- "neuralmagic/Meta-Llama-3-8B-Instruct-FP8",
14
+ "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
15
"nm-testing/Phi-3-mini-128k-instruct-FP8",
16
]
17
18
19
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
20
reason="FP8 is not supported on this GPU type.")
20
-@pytest.mark.parametrize("model", MODELS)
21
-def test_model_load_and_run(vllm_runner, model: str):
22
- with vllm_runner(model) as llm:
21
+@pytest.mark.parametrize("model_id", MODELS)
22
+def test_model_load_and_run(vllm_runner, model_id: str):
23
+ with vllm_runner(model_id) as llm:
24
+ # note: this does not test accuracy, just that we can run through
25
+ # see lm-eval tests for accuracy
26
+ outputs = llm.generate_greedy(prompts=["Hello my name is"],
27
+ max_tokens=10)
28
+ print(outputs[0][1])
29
+
30
+
31
+KV_CACHE_MODELS = [
32
+ # Deprecated AutoFP8 format using .kv_scale
33
+ "neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV",
34
+ # AutoFP8 format using separate .k_scale and .v_scale
35
+ "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",
36
+]
37
+
38
+
39
+@pytest.mark.skipif(not is_quant_method_supported("fp8"),
40
+ reason="FP8 is not supported on this GPU type.")
41
+@pytest.mark.parametrize("model_id", KV_CACHE_MODELS)
42
+def test_kv_cache_model_load_and_run(vllm_runner, model_id: str):
43
+ with vllm_runner(model_id, kv_cache_dtype="fp8") as llm:
44
+
45
+ model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
46
+ attn = model.model.layers[0].self_attn.attn
47
+ assert isinstance(attn.quant_method, Fp8KVCacheMethod)
48
+ # NOTE: it is valid for scales to be 1.0 (default value), but we know
49
+ # these checkpoints have scales < 1.0
50
+ assert 0.0 < attn._k_scale < 1.0
51
+ assert 0.0 < attn._v_scale < 1.0
52
+
53
# note: this does not test accuracy, just that we can run through
54
# see lm-eval tests for accuracy
55
outputs = llm.generate_greedy(prompts=["Hello my name is"],
vllm/_custom_ops.py
+11
-7
@@ -84,7 +84,8 @@ def paged_attention_v1(
84
max_seq_len: int,
85
alibi_slopes: Optional[torch.Tensor],
86
kv_cache_dtype: str,
87
- kv_scale: float,
87
+ k_scale: float,
88
+ v_scale: float,
89
tp_rank: int = 0,
90
blocksparse_local_blocks: int = 0,
91
blocksparse_vert_stride: int = 0,
@@ -94,8 +95,9 @@ def paged_attention_v1(
95
torch.ops._C.paged_attention_v1(
96
out, query, key_cache, value_cache, num_kv_heads, scale, block_tables,
97
seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
97
- kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride,
98
- blocksparse_block_size, blocksparse_head_sliding_step)
98
+ k_scale, v_scale, tp_rank, blocksparse_local_blocks,
99
+ blocksparse_vert_stride, blocksparse_block_size,
100
+ blocksparse_head_sliding_step)
101
102
103
def paged_attention_v2(
@@ -114,7 +116,8 @@ def paged_attention_v2(
116
max_seq_len: int,
117
alibi_slopes: Optional[torch.Tensor],
118
kv_cache_dtype: str,
117
- kv_scale: float,
119
+ k_scale: float,
120
+ v_scale: float,
121
tp_rank: int = 0,
122
blocksparse_local_blocks: int = 0,
123
blocksparse_vert_stride: int = 0,
@@ -124,7 +127,7 @@ def paged_attention_v2(
127
torch.ops._C.paged_attention_v2(
128
out, exp_sum, max_logits, tmp_out, query, key_cache, value_cache,
129
num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len,
127
- alibi_slopes, kv_cache_dtype, kv_scale, tp_rank,
130
+ alibi_slopes, kv_cache_dtype, k_scale, v_scale, tp_rank,
131
blocksparse_local_blocks, blocksparse_vert_stride,
132
blocksparse_block_size, blocksparse_head_sliding_step)
133
@@ -374,11 +377,12 @@ def reshape_and_cache(
377
value_cache: torch.Tensor,
378
slot_mapping: torch.Tensor,
379
kv_cache_dtype: str,
377
- kv_scale: float,
380
+ k_scale: float,
381
+ v_scale: float,
382
) -> None:
383
torch.ops._C_cache_ops.reshape_and_cache(key, value, key_cache,
384
value_cache, slot_mapping,
381
- kv_cache_dtype, kv_scale)
385
+ kv_cache_dtype, k_scale, v_scale)
386
387
388
def reshape_and_cache_flash(
vllm/_ipex_ops.py
+6
-3
@@ -59,7 +59,8 @@ class ipex_ops:
59
max_context_len: int,
60
alibi_slopes: Optional[torch.Tensor],
61
kv_cache_dtype: str,
62
- kv_scale: float,
62
+ k_scale: float,
63
+ v_scale: float,
64
tp_rank: int = 0,
65
blocksparse_local_blocks: int = 0,
66
blocksparse_vert_stride: int = 0,
@@ -99,7 +100,8 @@ class ipex_ops:
100
max_context_len: int,
101
alibi_slopes: Optional[torch.Tensor],
102
kv_cache_dtype: str,
102
- kv_scale: float,
103
+ k_scale: float,
104
+ v_scale: float,
105
tp_rank: int = 0,
106
blocksparse_local_blocks: int = 0,
107
blocksparse_vert_stride: int = 0,
@@ -227,7 +229,8 @@ class ipex_ops:
229
value_cache: torch.Tensor,
230
slot_mapping: torch.Tensor,
231
kv_cache_dtype: str,
230
- kv_scale: float,
232
+ k_scale: float,
233
+ v_scale: float,
234
) -> None:
235
assert kv_cache_dtype == "auto"
236
ipex.llm.modules.PagedAttention.reshape_and_cache(
vllm/attention/backends/abstract.py
+2
-1
@@ -134,7 +134,8 @@ class AttentionImpl(ABC, Generic[T]):
134
value: torch.Tensor,
135
kv_cache: torch.Tensor,
136
attn_metadata: T,
137
- kv_scale: float = 1.0,
137
+ k_scale: float = 1.0,
138
+ v_scale: float = 1.0,
139
attn_type: AttentionType = AttentionType.DECODER,
140
) -> torch.Tensor:
141
raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+6
-3
@@ -327,7 +327,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
327
value: torch.Tensor,
328
kv_cache: torch.Tensor,
329
attn_metadata: BlocksparseFlashAttentionMetadata,
330
- kv_scale: float = 1.0,
330
+ k_scale: float = 1.0,
331
+ v_scale: float = 1.0,
332
attn_type: AttentionType = AttentionType.DECODER,
333
) -> torch.Tensor:
334
"""Forward pass with FlashAttention and PagedAttention.
@@ -368,7 +369,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
369
value_cache,
370
attn_metadata.slot_mapping,
371
self.kv_cache_dtype,
371
- kv_scale,
372
+ k_scale,
373
+ v_scale,
374
)
375
376
if prefill_meta := attn_metadata.prefill_metadata:
@@ -405,7 +407,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
407
self.num_kv_heads,
408
self.scale,
409
self.alibi_slopes,
408
- kv_scale,
410
+ k_scale,
411
+ v_scale,
412
tp_rank=self.tp_rank,
413
blocksparse_local_blocks=self.local_blocks,
414
blocksparse_vert_stride=self.vert_stride,
vllm/attention/backends/flash_attn.py
+4
-2
@@ -256,7 +256,8 @@ class FlashAttentionImpl(AttentionImpl):
256
value: torch.Tensor,
257
kv_cache: torch.Tensor,
258
attn_metadata: FlashAttentionMetadata,
259
- kv_scale: float = 1.0,
259
+ k_scale: float = 1.0,
260
+ v_scale: float = 1.0,
261
attn_type: AttentionType = AttentionType.DECODER,
262
) -> torch.Tensor:
263
"""Forward pass with FlashAttention.
@@ -277,7 +278,8 @@ class FlashAttentionImpl(AttentionImpl):
278
"FlashAttentionImpl")
279
280
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
280
- assert kv_scale == 1.0, "kv_scale is not supported in FlashAttention."
281
+ assert k_scale == 1.0 and v_scale == 1.0, (
282
+ "key/v_scale is not supported in FlashAttention.")
283
284
num_tokens, hidden_size = query.shape
285
# Reshape the query, key, and value tensors.
vllm/attention/backends/flashinfer.py
+4
-2
@@ -223,10 +223,12 @@ class FlashInferImpl(AttentionImpl):
223
value: torch.Tensor,
224
kv_cache: Optional[torch.Tensor],
225
attn_metadata: FlashInferMetadata,
226
- kv_scale: float = 1.0,
226
+ k_scale: float = 1.0,
227
+ v_scale: float = 1.0,
228
attn_type: AttentionType = AttentionType.DECODER,
229
) -> torch.Tensor:
229
- assert kv_scale == 1.0
230
+ assert k_scale == 1.0 and v_scale == 1.0, (
231
+ "key/v_scale is not supported in FlashInfer.")
232
if attn_type != AttentionType.DECODER:
233
raise NotImplementedError("Encoder self-attention and "
234
"encoder/decoder cross-attention "
vllm/attention/backends/ipex_attn.py
+9
-5
@@ -156,7 +156,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
156
value: torch.Tensor,
157
kv_cache: Optional[torch.Tensor],
158
attn_metadata: IpexAttnMetadata, # type: ignore
159
- kv_scale: float = 1.0,
159
+ k_scale: float = 1.0,
160
+ v_scale: float = 1.0,
161
attn_type: AttentionType = AttentionType.DECODER,
162
) -> torch.Tensor:
163
"""Forward pass with IPEX varlen_attention and PagedAttention.
@@ -170,7 +171,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
171
Returns:
172
shape = [num_tokens, num_heads * head_size]
173
"""
173
- assert kv_scale == 1.0
174
+ assert k_scale == 1.0 and v_scale == 1.0
175
if attn_type != AttentionType.DECODER:
176
raise NotImplementedError("Encoder self-attention and "
177
"encoder/decoder cross-attention "
@@ -192,7 +193,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
193
value_cache,
194
attn_metadata.slot_mapping.flatten(),
195
self.kv_cache_dtype,
195
- kv_scale,
196
+ k_scale,
197
+ v_scale,
198
)
199
200
if attn_metadata.is_prompt:
@@ -273,7 +275,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
275
max_seq_len,
276
self.alibi_slopes,
277
self.kv_cache_dtype,
276
- kv_scale,
278
+ k_scale,
279
+ v_scale,
280
)
281
else:
282
# Run PagedAttention V2.
@@ -305,7 +308,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
308
max_seq_len,
309
self.alibi_slopes,
310
self.kv_cache_dtype,
308
- kv_scale,
311
+ k_scale,
312
+ v_scale,
313
)
314
315
# Reshape the output tensor.
vllm/attention/backends/pallas.py
+3
-2
@@ -131,7 +131,8 @@ class PallasAttentionBackendImpl(AttentionImpl):
131
value: torch.Tensor,
132
kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
133
attn_metadata: PallasMetadata,
134
- kv_scale: float = 1.0,
134
+ k_scale: float = 1.0,
135
+ v_scale: float = 1.0,
136
attn_type: AttentionType = AttentionType.DECODER,
137
) -> torch.Tensor:
138
"""Forward pass with Pallas attention.
@@ -146,7 +147,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
147
Returns:
148
shape = [batch_size, seq_len, num_heads * head_size]
149
"""
149
- assert kv_scale == 1.0
150
+ assert k_scale == 1.0 and v_scale == 1.0
151
if attn_type != AttentionType.DECODER:
152
raise NotImplementedError("Encoder self-attention and "
153
"encoder/decoder cross-attention "
vllm/attention/backends/rocm_flash_attn.py
+6
-3
@@ -296,7 +296,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
296
value: torch.Tensor,
297
kv_cache: torch.Tensor,
298
attn_metadata: ROCmFlashAttentionMetadata,
299
- kv_scale: float = 1.0,
299
+ k_scale: float = 1.0,
300
+ v_scale: float = 1.0,
301
attn_type: AttentionType = AttentionType.DECODER,
302
) -> torch.Tensor:
303
"""Forward pass with FlashAttention and PagedAttention.
@@ -336,7 +337,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
337
value_cache,
338
attn_metadata.slot_mapping,
339
self.kv_cache_dtype,
339
- kv_scale,
340
+ k_scale,
341
+ v_scale,
342
)
343
344
num_prefill_tokens = attn_metadata.num_prefill_tokens
@@ -456,7 +458,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
458
self.num_kv_heads,
459
self.scale,
460
self.alibi_slopes,
459
- kv_scale,
461
+ k_scale,
462
+ v_scale,
463
)
464
465
# Reshape the output tensor.
vllm/attention/backends/torch_sdpa.py
+7
-4
@@ -144,7 +144,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
144
value: torch.Tensor,
145
kv_cache: Optional[torch.Tensor],
146
attn_metadata: TorchSDPAMetadata, # type: ignore
147
- kv_scale: float = 1.0,
147
+ k_scale: float = 1.0,
148
+ v_scale: float = 1.0,
149
attn_type: AttentionType = AttentionType.DECODER,
150
) -> torch.Tensor:
151
"""Forward pass with torch SDPA and PagedAttention.
@@ -158,7 +159,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
159
Returns:
160
shape = [num_tokens, num_heads * head_size]
161
"""
161
- assert kv_scale == 1.0
162
+ assert k_scale == 1.0 and v_scale == 1.0
163
if attn_type != AttentionType.DECODER:
164
raise NotImplementedError("Encoder self-attention and "
165
"encoder/decoder cross-attention "
@@ -176,7 +177,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
177
PagedAttention.write_to_paged_cache(key, value, key_cache,
178
value_cache,
179
attn_metadata.slot_mapping,
179
- self.kv_cache_dtype, kv_scale)
180
+ self.kv_cache_dtype, k_scale,
181
+ v_scale)
182
183
if attn_metadata.is_prompt:
184
assert attn_metadata.seq_lens is not None
@@ -239,7 +241,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
241
self.num_kv_heads,
242
self.scale,
243
self.alibi_slopes,
242
- kv_scale,
244
+ k_scale,
245
+ v_scale,
246
)
247
248
# Reshape the output tensor.
vllm/attention/backends/xformers.py
+5
-3
@@ -427,7 +427,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
427
value: Optional[torch.Tensor],
428
kv_cache: Optional[torch.Tensor],
429
attn_metadata: "XFormersMetadata",
430
- kv_scale: float = 1.0,
430
+ k_scale: float = 1.0,
431
+ v_scale: float = 1.0,
432
attn_type: AttentionType = AttentionType.DECODER,
433
) -> torch.Tensor:
434
"""Forward pass with xFormers and PagedAttention.
@@ -531,7 +532,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
532
value_cache,
533
updated_slot_mapping,
534
self.kv_cache_dtype,
534
- kv_scale)
535
+ k_scale, v_scale)
536
537
if attn_type != AttentionType.ENCODER:
538
# Decoder self-attention supports chunked prefill.
@@ -620,7 +621,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
621
self.num_kv_heads,
622
self.scale,
623
self.alibi_slopes,
623
- kv_scale,
624
+ k_scale,
625
+ v_scale,
626
)
627
628
# Reshape the output tensor.
vllm/attention/layer.py
+8
-6
@@ -47,13 +47,14 @@ class Attention(nn.Module):
47
if num_kv_heads is None:
48
num_kv_heads = num_heads
49
50
- # The default kv_scale is set to 1.0. This is ignored
50
+ # The default k/v_scale is set to 1.0. This is ignored
51
# when kv-cache is not fp8, and should be used with
52
# kv-cache in fp8_e5m2. For kv-cache in fp8_e4m3, we
53
- # expect the pre-quantized kv_scale to be loaded along
53
+ # expect the pre-quantized k/v_scale to be loaded along
54
# with the model weights.
55
self.kv_cache_dtype = kv_cache_dtype
56
- self._kv_scale = 1.0
56
+ self._k_scale = 1.0
57
+ self._v_scale = 1.0
58
quant_method = quant_config.get_quant_method(
59
self) if quant_config else None
60
if quant_method is not None:
@@ -66,8 +67,8 @@ class Attention(nn.Module):
67
"fp8 checkpoints.")
68
# When FP8 quantization is enabled, we make a parameter
69
# "kv_scale" so that it can be loaded from FP8 checkpoint.
69
- # The kv_scale will then be converted back to self._kv_scale
70
- # in a native float32 value after weight loading.
70
+ # The k/v_scale will then be converted back to
71
+ # self._kv_scale in a native float32 value after weight loading
72
self.quant_method = quant_method
73
self.quant_method.create_weights(self)
74
@@ -98,7 +99,8 @@ class Attention(nn.Module):
99
value,
100
kv_cache,
101
attn_metadata,
101
- self._kv_scale,
102
+ self._k_scale,
103
+ self._v_scale,
104
attn_type=attn_type)
105
106
def extra_repr(self) -> str:
vllm/attention/ops/ipex_attn.py
+4
-2
@@ -45,7 +45,8 @@ class PagedAttention:
45
value_cache: torch.Tensor,
46
slot_mapping: torch.Tensor,
47
kv_cache_dtype: str,
48
- kv_scale: float,
48
+ k_scale: float,
49
+ v_scale: float,
50
*args,
51
) -> None:
52
ipex_modules.PagedAttention.reshape_and_cache(
@@ -64,7 +65,8 @@ class PagedAttention:
65
num_kv_heads: int,
66
scale: float,
67
alibi_slopes: Optional[torch.Tensor],
67
- kv_scale: float,
68
+ k_scale: float,
69
+ v_scale: float,
70
*args,
71
) -> torch.Tensor:
72
output = torch.empty_like(query)
vllm/attention/ops/paged_attn.py
+10
-5
@@ -66,7 +66,8 @@ class PagedAttention:
66
value_cache: torch.Tensor,
67
slot_mapping: torch.Tensor,
68
kv_cache_dtype: str,
69
- kv_scale: float,
69
+ k_scale: float,
70
+ v_scale: float,
71
) -> None:
72
ops.reshape_and_cache(
73
key,
@@ -75,7 +76,8 @@ class PagedAttention:
76
value_cache,
77
slot_mapping.flatten(),
78
kv_cache_dtype,
78
- kv_scale,
79
+ k_scale,
80
+ v_scale,
81
)
82
83
@staticmethod
@@ -90,7 +92,8 @@ class PagedAttention:
92
num_kv_heads: int,
93
scale: float,
94
alibi_slopes: Optional[torch.Tensor],
93
- kv_scale: float,
95
+ k_scale: float,
96
+ v_scale: float,
97
tp_rank: int = 0,
98
blocksparse_local_blocks: int = 0,
99
blocksparse_vert_stride: int = 0,
@@ -135,7 +138,8 @@ class PagedAttention:
138
max_seq_len,
139
alibi_slopes,
140
kv_cache_dtype,
138
- kv_scale,
141
+ k_scale,
142
+ v_scale,
143
tp_rank,
144
blocksparse_local_blocks,
145
blocksparse_vert_stride,
@@ -172,7 +176,8 @@ class PagedAttention:
176
max_seq_len,
177
alibi_slopes,
178
kv_cache_dtype,
175
- kv_scale,
179
+ k_scale,
180
+ v_scale,
181
tp_rank,
182
blocksparse_local_blocks,
183
blocksparse_vert_stride,
vllm/model_executor/layers/linear.py
+9
@@ -196,6 +196,15 @@ class ReplicatedLinear(LinearBase):
196
else:
197
self.register_parameter("bias", None)
198
199
+ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
200
+ # If the weight on disk does not have a shape, give it one
201
+ # (such scales for AutoFp8).
202
+ if len(loaded_weight.shape) == 0:
203
+ loaded_weight = loaded_weight.reshape(1)
204
+
205
+ assert param.size() == loaded_weight.size()
206
+ param.data.copy_(loaded_weight)
207
+
208
def forward(self, x: torch.Tensor) -> torch.Tensor:
209
bias = self.bias if not self.skip_bias_add else None
210
assert self.quant_method is not None
vllm/model_executor/layers/quantization/fp8.py
+38
-13
@@ -407,31 +407,56 @@ class Fp8KVCacheMethod(QuantizeMethodBase):
407
self.quant_config = quant_config
408
409
def create_weights(self, layer: torch.nn.Module):
410
- """Create "weight" (aka kv_scale) for an attention layer.
410
+ """Create "weight" (aka k_scale and v_scale) for an attention layer.
411
412
Args:
413
layer: The layer that is using the QuantizeMethodBase factory.
414
"""
415
- # Initialize the KV cache scale to 1.0 as the default value.
416
- # If the kv_scale appears in the checkpoint, it will be
415
+ # Initialize the KV cache scales to -1.0, which is an invalid value.
416
+ # If the k/v_scale appears in the checkpoint, it will be
417
# overwritten when loading weights.
418
- layer.kv_scale = Parameter(torch.tensor(1.0), requires_grad=False)
418
+ layer.k_scale = Parameter(torch.tensor(-1.0), requires_grad=False)
419
+ layer.v_scale = Parameter(torch.tensor(-1.0), requires_grad=False)
420
421
def apply(self, layer: torch.nn.Module) -> torch.Tensor:
422
raise RuntimeError("Fp8KVCacheMethod.apply should not be called.")
423
424
def process_weights_after_loading(self, layer: Module) -> None:
424
- # If the kv-cache dtype is auto, we enforce the kv-scale to be 1.0
425
+ # If the kv-cache dtype is auto, we enforce the k/v_scale to be 1.0
426
# regardless whether the kv-scale is available in the checkpoint.
427
if layer.kv_cache_dtype != "auto":
427
- kv_scale = layer.kv_scale.to("cpu").tolist()
428
- if not isinstance(kv_scale, float):
428
+ if layer.k_scale > 0.0 and layer.v_scale > 0.0:
429
+ # We prefer to use separate k_scale and v_scale if present
430
+ k_scale = layer.k_scale.to("cpu").tolist()
431
+ v_scale = layer.v_scale.to("cpu").tolist()
432
+ elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
433
+ # If no scales were loaded (both scales are invalid negative
434
+ # values), use the default value of 1.0
435
+ k_scale = Parameter(torch.tensor(1.0), requires_grad=False)
436
+ v_scale = Parameter(torch.tensor(1.0), requires_grad=False)
437
+ else:
438
+ # If we find a single kv_scale in the checkpoint, we remap
439
+ # kv_scale to k_scale during weight loading, and duplicate
440
+ # k_scale to v_scale here
441
+ assert layer.k_scale > 0.0
442
+ scale_to_duplicate = max(layer.k_scale, layer.v_scale)
443
+ k_scale = scale_to_duplicate.to("cpu").tolist()
444
+ v_scale = scale_to_duplicate.to("cpu").tolist()
445
+
446
+ if not isinstance(k_scale, float) or not isinstance(
447
+ v_scale, float):
448
raise ValueError("Only support per-tensor scaling factor "
449
"for fp8 KV cache")
431
- layer._kv_scale = kv_scale
432
- if layer._kv_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
450
+
451
+ # These are used in the final Attention.forward()
452
+ layer._k_scale = k_scale
453
+ layer._v_scale = v_scale
454
+ if (layer._k_scale == 1.0 and layer._v_scale == 1.0
455
+ and "e5m2" not in layer.kv_cache_dtype):
456
print_warning_once(
434
- "Using KV cache scaling factor 1.0 for fp8_e4m3. This may "
435
- "cause accuracy issues. Please make sure kv-cache scaling "
436
- "factor is available in the fp8 checkpoint.")
437
- del layer.kv_scale
457
+ "Using KV cache scaling factor 1.0 for fp8_e4m3. This "
458
+ "may cause accuracy issues. Please make sure k/v_scale "
459
+ "scaling factors are available in the fp8 checkpoint.")
460
+
461
+ del layer.k_scale
462
+ del layer.v_scale
vllm/model_executor/model_loader/weight_utils.py
+53
-5
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
22
from vllm.model_executor.layers.quantization import (QuantizationConfig,
23
get_quantization_config)
24
from vllm.model_executor.layers.quantization.schema import QuantParamSchema
25
+from vllm.utils import print_warning_once
26
27
logger = init_logger(__name__)
28
@@ -431,11 +432,6 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
432
def default_weight_loader(param: torch.Tensor,
433
loaded_weight: torch.Tensor) -> None:
434
"""Default weight loader."""
434
- # If the weight on disk does not have a shape, give it one
435
- # (such scales for AutoFp8).
436
- if len(loaded_weight.shape) == 0:
437
- loaded_weight = loaded_weight.reshape(1)
438
-
435
assert param.size() == loaded_weight.size()
436
param.data.copy_(loaded_weight)
437
@@ -462,3 +458,55 @@ def initialize_dummy_weights(
458
param.data.copy_(tmp_param)
459
else:
460
param.uniform_(low, high)
461
+
462
+
463
+def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> Optional[str]:
464
+ """Remap the name of FP8 k/v_scale parameters.
465
+
466
+ This function handles the remapping of FP8 k/v_scale parameter names.
467
+ It detects if the given name ends with a suffix and attempts to remap
468
+ it to the expected name format in the model. If the remapped name is not
469
+ found in the params_dict, a warning is printed and None is returned.
470
+
471
+ Args:
472
+ name (str): The original loaded checkpoint parameter name.
473
+ params_dict (dict): Dictionary containing the model's named parameters.
474
+
475
+ Returns:
476
+ str: The remapped parameter name if successful, or the original name
477
+ if no remapping is needed.
478
+ None: If the remapped name is not found in params_dict.
479
+ """
480
+ if name.endswith(".kv_scale"):
481
+ print_warning_once(
482
+ "DEPRECATED. Found kv_scale in the checkpoint. "
483
+ "This format is deprecated in favor of separate k_scale and "
484
+ "v_scale tensors and will be removed in a future release. "
485
+ "Functionally, we will remap kv_scale to k_scale and duplicate "
486
+ "k_scale to v_scale")
487
+ # NOTE: we remap the deprecated kv_scale to k_scale
488
+ remapped_name = name.replace(".kv_scale", ".attn.k_scale")
489
+ if remapped_name not in params_dict:
490
+ print_warning_once(
491
+ f"Found kv_scale in the checkpoint (e.g. {name}), "
492
+ "but not found the expected name in the model "
493
+ f"(e.g. {remapped_name}). kv_scale is "
494
+ "not loaded.")
495
+ return None
496
+ return remapped_name
497
+
498
+ possible_scale_names = [".k_scale", ".v_scale"]
499
+ for scale_name in possible_scale_names:
500
+ if name.endswith(scale_name):
501
+ remapped_name = name.replace(scale_name, f".attn{scale_name}")
502
+ if remapped_name not in params_dict:
503
+ print_warning_once(
504
+ f"Found {scale_name} in the checkpoint (e.g. {name}), "
505
+ "but not found the expected name in the model "
506
+ f"(e.g. {remapped_name}). {scale_name} is "
507
+ "not loaded.")
508
+ return None
509
+ return remapped_name
510
+
511
+ # If there were no matches, return the untouched param name
512
+ return name
vllm/model_executor/models/llama.py
+5
-14
@@ -44,10 +44,10 @@ from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
46
from vllm.model_executor.model_loader.weight_utils import (
47
- default_weight_loader, kv_cache_scales_loader)
47
+ default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
48
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.sequence import IntermediateTensors, SamplerOutput
50
-from vllm.utils import is_hip, print_warning_once
50
+from vllm.utils import is_hip
51
52
from .interfaces import SupportsLoRA
53
from .utils import is_pp_missing_parameter, make_layers
@@ -460,18 +460,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
460
if name.endswith(".bias") and name not in params_dict:
461
continue
462
# Remapping the name of FP8 kv-scale.
463
- if name.endswith("kv_scale"):
464
- remapped_kv_scale_name = name.replace(
465
- ".kv_scale", ".attn.kv_scale")
466
- if remapped_kv_scale_name not in params_dict:
467
- print_warning_once(
468
- f"Found kv scale in the checkpoint (e.g. {name}), "
469
- "but not found the expected name in the model "
470
- f"(e.g. {remapped_kv_scale_name}). kv-scale is "
471
- "not loaded.")
472
- continue
473
- else:
474
- name = remapped_kv_scale_name
463
+ name = maybe_remap_kv_scale_name(name, params_dict)
464
+ if name is None:
465
+ continue
466
467
if is_pp_missing_parameter(name, self):
468
continue
vllm/model_executor/models/mixtral.py
+6
-15
@@ -42,10 +42,10 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
+from vllm.model_executor.model_loader.weight_utils import (
46
+ default_weight_loader, maybe_remap_kv_scale_name)
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.sequence import IntermediateTensors, SamplerOutput
48
-from vllm.utils import print_warning_once
49
50
from .interfaces import SupportsLoRA
51
@@ -415,19 +415,10 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
415
if name.endswith(".bias") and name not in params_dict:
416
continue
417
# Remapping the name of FP8 kv-scale.
418
- if name.endswith("kv_scale"):
419
- remapped_kv_scale_name = name.replace(
420
- ".kv_scale", ".attn.kv_scale")
421
- if remapped_kv_scale_name not in params_dict:
422
- print_warning_once(
423
- "Found kv scale in the checkpoint "
424
- f"(e.g. {name}), but not found the expected "
425
- f"name in the model "
426
- f"(e.g. {remapped_kv_scale_name}). "
427
- "kv-scale is not loaded.")
428
- continue
429
- else:
430
- name = remapped_kv_scale_name
418
+ name = maybe_remap_kv_scale_name(name, params_dict)
419
+ if name is None:
420
+ continue
421
+
422
param = params_dict[name]
423
weight_loader = getattr(param, "weight_loader",
424
default_weight_loader)
vllm/model_executor/models/qwen2.py
+6
-14
@@ -43,10 +43,10 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
ParallelLMHead, VocabParallelEmbedding)
46
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
+from vllm.model_executor.model_loader.weight_utils import (
47
+ default_weight_loader, maybe_remap_kv_scale_name)
48
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.sequence import IntermediateTensors, SamplerOutput
49
-from vllm.utils import print_warning_once
50
51
from .interfaces import SupportsLoRA
52
@@ -382,18 +382,10 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
382
if name.endswith(".bias") and name not in params_dict:
383
continue
384
# Remapping the name of FP8 kv-scale.
385
- if name.endswith("kv_scale"):
386
- remapped_kv_scale_name = name.replace(
387
- ".kv_scale", ".attn.kv_scale")
388
- if remapped_kv_scale_name not in params_dict:
389
- print_warning_once(
390
- f"Found kv scale in the checkpoint (e.g. {name}), "
391
- "but not found the expected name in the model "
392
- f"(e.g. {remapped_kv_scale_name}). kv-scale is "
393
- "not loaded.")
394
- continue
395
- else:
396
- name = remapped_kv_scale_name
385
+ name = maybe_remap_kv_scale_name(name, params_dict)
386
+ if name is None:
387
+ continue
388
+
389
param = params_dict[name]
390
weight_loader = getattr(param, "weight_loader",
391
default_weight_loader)