A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[FP8][Kernel] Dynamic kv cache scaling factors computation (#11906)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com> Co-authored-by: Micah Williamson <micah.williamson@amd.com>
Gregory Shtrasberg committed
Jan 23, 2025 at 13:04 UTC
e97f802b2d74861af77997691a7d1c36498f6dca
60 files changed
+276
-1365
benchmarks/kernels/benchmark_paged_attention.py
+3
-1
@@ -98,7 +98,9 @@ def main(
98
start_time = time.perf_counter()
99
100
# Using default kv_scale
101
- k_scale = v_scale = 1.0
101
+ k_scale = v_scale = torch.tensor(1.0,
102
+ dtype=torch.float32,
103
+ device=device)
104
105
for _ in range(num_iters):
106
if version == "v1":
csrc/attention/attention_kernels.cuh
+5
-5
@@ -105,7 +105,7 @@ __device__ void paged_attention_kernel(
105
const int max_num_blocks_per_seq,
106
const float* __restrict__ alibi_slopes, // [num_heads]
107
const int q_stride, const int kv_block_stride, const int kv_head_stride,
108
- const float k_scale, const float v_scale, const int tp_rank,
108
+ const float* k_scale, const float* v_scale, const int tp_rank,
109
const int blocksparse_local_blocks, const int blocksparse_vert_stride,
110
const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
111
const int seq_idx = blockIdx.y;
@@ -285,7 +285,7 @@ __device__ void paged_attention_kernel(
285
Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
286
k_ptr + offset1 * BLOCK_SIZE * x + offset2);
287
k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(
288
- k_vec_quant, k_scale);
288
+ k_vec_quant, *k_scale);
289
}
290
}
291
@@ -415,7 +415,7 @@ __device__ void paged_attention_kernel(
415
*reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
416
// Vector conversion from V_quant_vec to V_vec.
417
v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec,
418
- v_scale);
418
+ *v_scale);
419
}
420
if (block_idx == num_seq_blocks - 1) {
421
// NOTE(woosuk): When v_vec contains the tokens that are out of the
@@ -513,7 +513,7 @@ __global__ void paged_attention_v1_kernel(
513
const int max_num_blocks_per_seq,
514
const float* __restrict__ alibi_slopes, // [num_heads]
515
const int q_stride, const int kv_block_stride, const int kv_head_stride,
516
- const float k_scale, const float v_scale, const int tp_rank,
516
+ const float* k_scale, const float* v_scale, const int tp_rank,
517
const int blocksparse_local_blocks, const int blocksparse_vert_stride,
518
const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
519
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
@@ -549,7 +549,7 @@ __global__ void paged_attention_v2_kernel(
549
const int max_num_blocks_per_seq,
550
const float* __restrict__ alibi_slopes, // [num_heads]
551
const int q_stride, const int kv_block_stride, const int kv_head_stride,
552
- const float k_scale, const float v_scale, const int tp_rank,
552
+ const float* k_scale, const float* v_scale, const int tp_rank,
553
const int blocksparse_local_blocks, const int blocksparse_vert_stride,
554
const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
555
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
csrc/attention/paged_attention_v1.cu
+10
-7
@@ -41,7 +41,7 @@
41
out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
42
scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
43
alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
44
- k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
44
+ k_scale_ptr, v_scale_ptr, tp_rank, blocksparse_local_blocks, \
45
blocksparse_vert_stride, blocksparse_block_size, \
46
blocksparse_head_sliding_step);
47
@@ -53,10 +53,10 @@ void paged_attention_v1_launcher(
53
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
54
torch::Tensor& value_cache, int num_kv_heads, float scale,
55
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
56
- const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
57
- float v_scale, const int tp_rank, const int blocksparse_local_blocks,
58
- const int blocksparse_vert_stride, const int blocksparse_block_size,
59
- const int blocksparse_head_sliding_step) {
56
+ const std::optional<torch::Tensor>& alibi_slopes, torch::Tensor& k_scale,
57
+ torch::Tensor& v_scale, const int tp_rank,
58
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
59
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
60
int num_seqs = query.size(0);
61
int num_heads = query.size(1);
62
int head_size = query.size(2);
@@ -80,6 +80,8 @@ void paged_attention_v1_launcher(
80
CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
81
int* block_tables_ptr = block_tables.data_ptr<int>();
82
int* seq_lens_ptr = seq_lens.data_ptr<int>();
83
+ const float* k_scale_ptr = reinterpret_cast<const float*>(k_scale.data_ptr());
84
+ const float* v_scale_ptr = reinterpret_cast<const float*>(v_scale.data_ptr());
85
86
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
87
int padded_max_seq_len =
@@ -177,8 +179,9 @@ void paged_attention_v1(
179
torch::Tensor& seq_lens, // [num_seqs]
180
int64_t block_size, int64_t max_seq_len,
181
const std::optional<torch::Tensor>& alibi_slopes,
180
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
181
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
182
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
183
+ torch::Tensor& v_scale, const int64_t tp_rank,
184
+ const int64_t blocksparse_local_blocks,
185
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
186
const int64_t blocksparse_head_sliding_step) {
187
const bool is_block_sparse = (blocksparse_vert_stride > 1);
csrc/attention/paged_attention_v2.cu
+10
-7
@@ -37,7 +37,7 @@
37
exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
38
value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
39
seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
40
- kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
40
+ kv_block_stride, kv_head_stride, k_scale_ptr, v_scale_ptr, tp_rank, \
41
blocksparse_local_blocks, blocksparse_vert_stride, \
42
blocksparse_block_size, blocksparse_head_sliding_step); \
43
vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
@@ -54,10 +54,10 @@ void paged_attention_v2_launcher(
54
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
55
torch::Tensor& value_cache, int num_kv_heads, float scale,
56
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
57
- const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
58
- float v_scale, const int tp_rank, const int blocksparse_local_blocks,
59
- const int blocksparse_vert_stride, const int blocksparse_block_size,
60
- const int blocksparse_head_sliding_step) {
57
+ const std::optional<torch::Tensor>& alibi_slopes, torch::Tensor& k_scale,
58
+ torch::Tensor& v_scale, const int tp_rank,
59
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
60
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
61
int num_seqs = query.size(0);
62
int num_heads = query.size(1);
63
int head_size = query.size(2);
@@ -84,6 +84,8 @@ void paged_attention_v2_launcher(
84
CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
85
int* block_tables_ptr = block_tables.data_ptr<int>();
86
int* seq_lens_ptr = seq_lens.data_ptr<int>();
87
+ const float* k_scale_ptr = reinterpret_cast<const float*>(k_scale.data_ptr());
88
+ const float* v_scale_ptr = reinterpret_cast<const float*>(v_scale.data_ptr());
89
90
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
91
int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
@@ -188,8 +190,9 @@ void paged_attention_v2(
190
torch::Tensor& seq_lens, // [num_seqs]
191
int64_t block_size, int64_t max_seq_len,
192
const std::optional<torch::Tensor>& alibi_slopes,
191
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
192
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
193
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
194
+ torch::Tensor& v_scale, const int64_t tp_rank,
195
+ const int64_t blocksparse_local_blocks,
196
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
197
const int64_t blocksparse_head_sliding_step) {
198
const bool is_block_sparse = (blocksparse_vert_stride > 1);
csrc/cache.h
+3
-3
@@ -18,15 +18,15 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
18
void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
19
torch::Tensor& key_cache, torch::Tensor& value_cache,
20
torch::Tensor& slot_mapping,
21
- const std::string& kv_cache_dtype, const double k_scale,
22
- const double v_scale);
21
+ const std::string& kv_cache_dtype,
22
+ torch::Tensor& k_scale, torch::Tensor& v_scale);
23
24
void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
25
torch::Tensor& key_cache,
26
torch::Tensor& value_cache,
27
torch::Tensor& slot_mapping,
28
const std::string& kv_cache_dtype,
29
- const double k_scale, const double v_scale);
29
+ torch::Tensor& k_scale, torch::Tensor& v_scale);
30
31
// Just for unittest
32
void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
csrc/cache_kernels.cu
+17
-13
@@ -159,8 +159,8 @@ __global__ void reshape_and_cache_kernel(
159
// block_size]
160
const int64_t* __restrict__ slot_mapping, // [num_tokens]
161
const int key_stride, const int value_stride, const int num_heads,
162
- const int head_size, const int block_size, const int x, const float k_scale,
163
- const float v_scale) {
162
+ const int head_size, const int block_size, const int x,
163
+ const float* k_scale, const float* v_scale) {
164
const int64_t token_idx = blockIdx.x;
165
const int64_t slot_idx = slot_mapping[token_idx];
166
if (slot_idx < 0) {
@@ -196,9 +196,9 @@ __global__ void reshape_and_cache_kernel(
196
value_cache[tgt_value_idx] = tgt_value;
197
} else {
198
key_cache[tgt_key_idx] =
199
- fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, k_scale);
199
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, *k_scale);
200
value_cache[tgt_value_idx] =
201
- fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, v_scale);
201
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, *v_scale);
202
}
203
}
204
}
@@ -214,7 +214,7 @@ __global__ void reshape_and_cache_flash_kernel(
214
const int64_t* __restrict__ slot_mapping, // [num_tokens]
215
const int block_stride, const int key_stride, const int value_stride,
216
const int num_heads, const int head_size, const int block_size,
217
- const float k_scale, const float v_scale) {
217
+ const float* k_scale, const float* v_scale) {
218
const int64_t token_idx = blockIdx.x;
219
const int64_t slot_idx = slot_mapping[token_idx];
220
// NOTE: slot_idx can be -1 if the token is padded
@@ -239,9 +239,9 @@ __global__ void reshape_and_cache_flash_kernel(
239
value_cache[tgt_key_value_idx] = tgt_value;
240
} else {
241
key_cache[tgt_key_value_idx] =
242
- fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, k_scale);
242
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, *k_scale);
243
value_cache[tgt_key_value_idx] =
244
- fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, v_scale);
244
+ fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, *v_scale);
245
}
246
}
247
}
@@ -258,7 +258,9 @@ __global__ void reshape_and_cache_flash_kernel(
258
reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
259
reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
260
slot_mapping.data_ptr<int64_t>(), key_stride, value_stride, \
261
- num_heads, head_size, block_size, x, k_scale, v_scale);
261
+ num_heads, head_size, block_size, x, \
262
+ reinterpret_cast<const float*>(k_scale.data_ptr()), \
263
+ reinterpret_cast<const float*>(v_scale.data_ptr()));
264
265
void reshape_and_cache(
266
torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -268,8 +270,8 @@ void reshape_and_cache(
270
torch::Tensor&
271
value_cache, // [num_blocks, num_heads, head_size, block_size]
272
torch::Tensor& slot_mapping, // [num_tokens]
271
- const std::string& kv_cache_dtype, const double k_scale,
272
- const double v_scale) {
273
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
274
+ torch::Tensor& v_scale) {
275
int num_tokens = key.size(0);
276
int num_heads = key.size(1);
277
int head_size = key.size(2);
@@ -299,7 +301,9 @@ void reshape_and_cache(
301
reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
302
reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
303
slot_mapping.data_ptr<int64_t>(), block_stride, key_stride, \
302
- value_stride, num_heads, head_size, block_size, k_scale, v_scale);
304
+ value_stride, num_heads, head_size, block_size, \
305
+ reinterpret_cast<const float*>(k_scale.data_ptr()), \
306
+ reinterpret_cast<const float*>(v_scale.data_ptr()));
307
308
void reshape_and_cache_flash(
309
torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -308,8 +312,8 @@ void reshape_and_cache_flash(
312
torch::Tensor&
313
value_cache, // [num_blocks, block_size, num_heads, head_size]
314
torch::Tensor& slot_mapping, // [num_tokens] or [num_actual_tokens]
311
- const std::string& kv_cache_dtype, const double k_scale,
312
- const double v_scale) {
315
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
316
+ torch::Tensor& v_scale) {
317
// NOTE(woosuk): In vLLM V1, key.size(0) can be different from
318
// slot_mapping.size(0) because of padding for CUDA graphs.
319
// In vLLM V0, key.size(0) is always equal to slot_mapping.size(0) because
csrc/cpu/attention.cpp
+6
-6
@@ -460,11 +460,11 @@ void paged_attention_v1(
460
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
461
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
462
int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
463
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
464
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
463
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
464
+ torch::Tensor& v_scale, const int64_t tp_rank,
465
+ const int64_t blocksparse_local_blocks,
466
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
467
const int64_t blocksparse_head_sliding_step) {
467
- TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
468
TORCH_CHECK(blocksparse_vert_stride <= 1,
469
"CPU backend does not support blocksparse attention yet.");
470
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
@@ -782,11 +782,11 @@ void paged_attention_v2(
782
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
783
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
784
int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
785
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
786
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
785
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
786
+ torch::Tensor& v_scale, const int64_t tp_rank,
787
+ const int64_t blocksparse_local_blocks,
788
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
789
const int64_t blocksparse_head_sliding_step) {
789
- TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
790
TORCH_CHECK(blocksparse_vert_stride <= 1,
791
"CPU backend does not support blocksparse attention yet.");
792
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
csrc/cpu/cache.cpp
+2
-4
@@ -107,10 +107,8 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
107
void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
108
torch::Tensor& key_cache, torch::Tensor& value_cache,
109
torch::Tensor& slot_mapping,
110
- const std::string& kv_cache_dtype, double k_scale,
111
- double v_scale) {
112
- TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
113
-
110
+ const std::string& kv_cache_dtype,
111
+ torch::Tensor& k_scale, torch::Tensor& v_scale) {
112
int num_tokens = key.size(0);
113
int num_heads = key.size(1);
114
int head_size = key.size(2);
csrc/cpu/torch_bindings.cpp
+3
-3
@@ -30,7 +30,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
30
" Tensor value_cache, int num_kv_heads, float scale,"
31
" Tensor block_tables, Tensor seq_lens, int block_size,"
32
" int max_seq_len, Tensor? alibi_slopes,"
33
- " str kv_cache_dtype, float k_scale, float v_scale,"
33
+ " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
34
" int tp_rank, int blocksparse_local_blocks,"
35
" int blocksparse_vert_stride, int blocksparse_block_size,"
36
" int blocksparse_head_sliding_step) -> ()");
@@ -44,7 +44,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
44
" Tensor value_cache, int num_kv_heads, float scale,"
45
" Tensor block_tables, Tensor seq_lens, int block_size,"
46
" int max_seq_len, Tensor? alibi_slopes,"
47
- " str kv_cache_dtype, float k_scale, float v_scale,"
47
+ " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
48
" int tp_rank, int blocksparse_local_blocks,"
49
" int blocksparse_vert_stride, int blocksparse_block_size,"
50
" int blocksparse_head_sliding_step) -> ()");
@@ -148,7 +148,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
148
" Tensor! key_cache, Tensor! value_cache,"
149
" Tensor slot_mapping,"
150
" str kv_cache_dtype,"
151
- " float k_scale, float v_scale) -> ()");
151
+ " Tensor k_scale, Tensor v_scale) -> ()");
152
cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
153
}
154
csrc/ops.h
+6
-4
@@ -34,8 +34,9 @@ void paged_attention_v1(
34
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
35
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
36
int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
37
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
38
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
37
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
38
+ torch::Tensor& v_scale, const int64_t tp_rank,
39
+ const int64_t blocksparse_local_blocks,
40
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
41
const int64_t blocksparse_head_sliding_step);
42
@@ -45,8 +46,9 @@ void paged_attention_v2(
46
torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
47
torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
48
int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
48
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
49
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
49
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
50
+ torch::Tensor& v_scale, const int64_t tp_rank,
51
+ const int64_t blocksparse_local_blocks,
52
const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
53
const int64_t blocksparse_head_sliding_step);
54
csrc/rocm/attention.cu
+10
-7
@@ -218,7 +218,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
218
scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
219
// head_size]
220
scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
221
- int max_ctx_blocks, float k_scale, float v_scale) {
221
+ int max_ctx_blocks, const float* k_scale_ptr, const float* v_scale_ptr) {
222
constexpr int NWARPS = NUM_THREADS / WARP_SIZE;
223
const int warpid = threadIdx.x / WARP_SIZE;
224
const int laneid = threadIdx.x % WARP_SIZE;
@@ -406,7 +406,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
406
// Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
407
const _B8x8 Vlocalb8 = v_ptrh8be[d];
408
Vlocal[h][b * BLOCK_SIZE / 8 + d] =
409
- scaled_convert_b8x8<scalar_t, KV_DTYPE>(Vlocalb8, v_scale);
409
+ scaled_convert_b8x8<scalar_t, KV_DTYPE>(Vlocalb8, *v_scale_ptr);
410
}
411
}
412
}
@@ -416,7 +416,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
416
#pragma unroll
417
for (int d = 0; d < KHELOOP; d++) {
418
Klocal[d] =
419
- scaled_convert_b8x8<scalar_t, KV_DTYPE>(Klocalb8[d], k_scale);
419
+ scaled_convert_b8x8<scalar_t, KV_DTYPE>(Klocalb8[d], *k_scale_ptr);
420
}
421
}
422
@@ -890,7 +890,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
890
scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
891
// head_size]
892
scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
893
- int max_ctx_blocks, float k_scale, float v_scale) {
893
+ int max_ctx_blocks, const float* k_scale, const float* v_scale) {
894
UNREACHABLE_CODE
895
}
896
@@ -919,7 +919,7 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
919
block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
920
alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
921
exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks, \
922
- k_scale, v_scale);
922
+ k_scale_ptr, v_scale_ptr);
923
924
template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
925
int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 512>
@@ -929,7 +929,7 @@ void paged_attention_custom_launcher(
929
torch::Tensor& value_cache, const int num_kv_heads, float scale,
930
torch::Tensor& block_tables, torch::Tensor& context_lens,
931
int max_context_len, const std::optional<torch::Tensor>& alibi_slopes,
932
- float k_scale, float v_scale) {
932
+ torch::Tensor& k_scale, torch::Tensor& v_scale) {
933
int num_seqs = query.size(0);
934
int num_heads = query.size(1);
935
int head_size = query.size(2);
@@ -953,6 +953,8 @@ void paged_attention_custom_launcher(
953
KVT* value_cache_ptr = reinterpret_cast<KVT*>(value_cache.data_ptr());
954
int* block_tables_ptr = block_tables.data_ptr<int>();
955
int* context_lens_ptr = context_lens.data_ptr<int>();
956
+ const float* k_scale_ptr = reinterpret_cast<const float*>(k_scale.data_ptr());
957
+ const float* v_scale_ptr = reinterpret_cast<const float*>(v_scale.data_ptr());
958
959
const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE);
960
const int max_num_partitions =
@@ -1087,7 +1089,8 @@ void paged_attention(
1089
torch::Tensor& context_lens, // [num_seqs]
1090
int64_t block_size, int64_t max_context_len,
1091
const std::optional<torch::Tensor>& alibi_slopes,
1090
- const std::string& kv_cache_dtype, double k_scale, double v_scale) {
1092
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
1093
+ torch::Tensor& v_scale) {
1094
const int head_size = query.size(2);
1095
if (kv_cache_dtype == "auto") {
1096
if (query.dtype() == at::ScalarType::Half) {
csrc/rocm/ops.h
+2
-2
@@ -10,5 +10,5 @@ void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
10
torch::Tensor& context_lens, int64_t block_size,
11
int64_t max_context_len,
12
const std::optional<torch::Tensor>& alibi_slopes,
13
- const std::string& kv_cache_dtype, double k_scale,
14
- double v_scale);
13
+ const std::string& kv_cache_dtype, torch::Tensor& k_scale,
14
+ torch::Tensor& v_scale);
csrc/rocm/torch_bindings.cpp
+1
-1
@@ -27,7 +27,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
27
" int max_context_len,"
28
" Tensor? alibi_slopes,"
29
" str kv_cache_dtype,"
30
- " float k_scale, float v_scale) -> ()");
30
+ " Tensor k_scale, Tensor v_scale) -> ()");
31
rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
32
}
33
csrc/torch_bindings.cpp
+4
-4
@@ -30,7 +30,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
30
" Tensor value_cache, int num_kv_heads, float scale,"
31
" Tensor block_tables, Tensor seq_lens, int block_size,"
32
" int max_seq_len, Tensor? alibi_slopes,"
33
- " str kv_cache_dtype, float k_scale, float v_scale,"
33
+ " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
34
" int tp_rank, int blocksparse_local_blocks,"
35
" int blocksparse_vert_stride, int blocksparse_block_size,"
36
" int blocksparse_head_sliding_step) -> ()");
@@ -44,7 +44,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
44
" Tensor value_cache, int num_kv_heads, float scale,"
45
" Tensor block_tables, Tensor seq_lens, int block_size,"
46
" int max_seq_len, Tensor? alibi_slopes,"
47
- " str kv_cache_dtype, float k_scale, float v_scale,"
47
+ " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
48
" int tp_rank, int blocksparse_local_blocks,"
49
" int blocksparse_vert_stride, int blocksparse_block_size,"
50
" int blocksparse_head_sliding_step) -> ()");
@@ -449,7 +449,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
449
" Tensor! key_cache, Tensor! value_cache,"
450
" Tensor slot_mapping,"
451
" str kv_cache_dtype,"
452
- " float k_scale, float v_scale) -> ()");
452
+ " Tensor k_scale, Tensor v_scale) -> ()");
453
cache_ops.impl("reshape_and_cache", torch::kCUDA, &reshape_and_cache);
454
455
// Reshape the key and value tensors and cache them.
@@ -459,7 +459,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
459
" Tensor! value_cache,"
460
" Tensor slot_mapping,"
461
" str kv_cache_dtype,"
462
- " float k_scale, float v_scale) -> ()");
462
+ " Tensor k_scale, Tensor v_scale) -> ()");
463
cache_ops.impl("reshape_and_cache_flash", torch::kCUDA,
464
&reshape_and_cache_flash);
465
docs/source/features/quantization/quantized_kvcache.md
+6
-4
@@ -35,16 +35,18 @@ Studies have shown that FP8 E4M3 quantization typically only minimally degrades
35
Here is an example of how to enable FP8 quantization:
36
37
```python
38
+# To calculate kv cache scales on the fly enable the calculate_kv_scales
39
+# parameter
40
+
41
from vllm import LLM, SamplingParams
42
43
sampling_params = SamplingParams(temperature=0.7, top_p=0.8)
41
-llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct", kv_cache_dtype="fp8")
44
+llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",
45
+ kv_cache_dtype="fp8",
46
+ calculate_kv_scales=True)
47
prompt = "London is the capital of"
48
out = llm.generate(prompt, sampling_params)[0].outputs[0].text
49
print(out)
45
-
46
-# output w/ scaling factors: England, the United Kingdom, and one of the world's leading financial,
47
-# output w/o scaling factors: England, located in the southeastern part of the country. It is known
50
```
51
52
The `kv_cache_dtype` argument specifies the data type for KV cache storage:
examples/other/fp8/README.md
deleted
-96
@@ -1,96 +0,0 @@
1
-# FP8 KV Cache
2
-
3
-This utility extracts the KV cache scaling factors from a quantized HF (Hugging Face) model. The extracted scaling factors are saved to a JSON file, which can later be used by vLLM (variable-length language model) during runtime. This tool is particularly useful when the KV cache data type is FP8 and is intended for use on ROCm (AMD GPU) platforms.
4
-
5
-## Prerequisites
6
-
7
-- Python 3.x
8
-- PyTorch
9
-- NumPy
10
-- Hugging Face Transformers
11
-- Hugging Face Hub
12
-- AMMO
13
-
14
-Before incorporating the FP8 datatype for inference workloads, you must adhere to the following steps:
15
-1. Install all necessary prerequisites and dependencies.
16
-2. Convert HF model into a quantized HF model.
17
-3. Extract KV Cache Scaling Factors from quantized HF model.
18
-4. Load KV Cache Scaling Factors into VLLM.
19
-
20
-### 2. Convert HF model into a quantized HF model.
21
-Note: The following steps are adapted from the [TensorRT-LLM repository](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/quantization/README.md).
22
-
23
-`quantize.py` (examples/other/fp8/quantizer/quantize.py) uses the quantization toolkit (AMMO) to calibrate the PyTorch models and export TensorRT-LLM checkpoints. Each TensorRT-LLM checkpoint contains a config file (in .json format) and one or several rank weight files (in .safetensors format).
24
-
25
-The detailed quantization toolkit (AMMO) conversion guide for FP8 can be found at `examples/other/fp8/quantizer/README.md`.
26
-
27
-### 3. Extract KV Cache Scaling Factors from quantized HF model.
28
-`extract_scales.py` (examples/other/fp8/extract_scales.py) can be utilized to extract the KV cache scaling factors from your quantized HF model, however at the moment, this tool exclusively supports Llama 2 models. It is also important to note the following:
29
-1. **File Structure**: The utility operates under the assumption that all parameters, including KV cache scaling factors, corresponding to a particular Tensor Parallelism (TP) rank are stored in a single file. These files must adhere to a specific naming convention where the TP rank is immediately identified after a specific keyword (e.g., "rank") in the filename.
30
-
31
-2. **TP Decomposition**: The utility assumes consistency between the TP decomposition employed by the quantizer tool and that used by vLLM.
32
-
33
-3. **AMMO Compatibility**: Currently, the generated KV cache scaling factors for AMMO remain uniform across all TP ranks.
34
-
35
-```python
36
-# prerequisites:
37
-# - Quantized HF LLaMa 2 model
38
-python3 examples/other/fp8/extract_scales.py --help
39
-Usage: extract_scales.py [-h] --quantized_model QUANTIZED_MODEL [--load_format {auto,safetensors,npz,pt}] [--output_dir OUTPUT_DIR] [--output_name OUTPUT_NAME] [--tp_size TP_SIZE]
40
-
41
-KV Scale Extraction Example
42
-
43
-optional arguments:
44
---quantized_model: Specify either the local path to, or name of, a quantized HF model. It is expected that the quantization format is FP8_E4M3, for use on ROCm (AMD GPU).
45
-Optional arguments:
46
---cache_dir: Specify a cache directory to use in the event of a HF model download. (Default: None)
47
---load_format: Specify the format of the model's tensor files containing the KV cache scaling factors. (Choices: auto, safetensors, npz, pt; Default: auto)
48
---revision: Specify the model's revision number. (Default: None)
49
---output_dir: Specify the output directory. By default the KV cache scaling factors will be saved in the model directory. (Default: None)
50
---output_name: Specify the output filename. (Default: kv_cache_scales.json)
51
---tp_size: Specify the tensor-parallel (TP) size that the quantized model should correspond to. If specified, during KV cache scaling factor extraction the observed TP size will be checked against this and an error will be raised if there is a mismatch. (Default: None)
52
-```
53
-```python
54
-Example:
55
-python3 examples/other/fp8/extract_scales.py --quantized_model <QUANTIZED_MODEL_DIR> --tp_size <TENSOR_PARALLEL_SIZE> --output_dir <PATH_TO_OUTPUT_DIR>
56
-```
57
-### 4. Load KV Cache Scaling Factors into VLLM.
58
-This script evaluates the inference throughput of language models using various backends such as vLLM. It measures the time taken to process a given number of prompts and generate sequences for each prompt. The recently generated KV cache scaling factors are now integrated into the benchmarking process and allow for KV cache scaling factors to be utilized for FP8.
59
-```
60
-# prerequisites:
61
-# - LLaMa 2 kv_cache_scales.json file
62
-
63
-python3 benchmarks/benchmark_throughput.py --help
64
-usage: benchmark_throughput.py [-h] [--backend {vllm,hf,mii}] [--dataset DATASET] [--input-len INPUT_LEN] [--output-len OUTPUT_LEN] [--model MODEL]
65
- [--tokenizer TOKENIZER] [--quantization {awq,gptq,None}] [--tensor-parallel-size TENSOR_PARALLEL_SIZE] [--n N]
66
- [--use-beam-search] [--num-prompts NUM_PROMPTS] [--seed SEED] [--hf-max-batch-size HF_MAX_BATCH_SIZE] [--trust-remote-code]
67
- [--max-model-len MAX_MODEL_LEN] [--dtype {auto,half,float16,bfloat16,float,float32}] [--enforce-eager] [--kv-cache-dtype {auto,fp8}]
68
- [--quantization-param-path KV_CACHE_quantization_param_path]
69
-
70
-Benchmark Throughput Example
71
-optional arguments:
72
- -h, --help show this help message and exit
73
- --backend {vllm,hf,mii}
74
- --dataset DATASET Path to the dataset.
75
- --input-len INPUT_LEN Input prompt length for each request
76
- --output-len OUTPUT_LEN Output length for each request. Overrides the output length from the dataset.
77
- --model MODEL
78
- --tokenizer TOKENIZER
79
- --quantization {awq,gptq,None}, -q {awq,gptq,None}
80
- --tensor-parallel-size TENSOR_PARALLEL_SIZE, -tp TENSOR_PARALLEL_SIZE
81
- --n N Number of generated sequences per prompt.
82
- --use-beam-search
83
- --num-prompts NUM_PROMPTS Number of prompts to process.
84
- --seed SEED
85
- --hf-max-batch-size HF_MAX_BATCH_SIZE Maximum batch size for HF backend.
86
- --trust-remote-code trust remote code from huggingface
87
- --max-model-len MAX_MODEL_LEN Maximum length of a sequence (including prompt and output). If None, will be derived from the model.
88
- --dtype {auto,half,float16,bfloat16,float,float32} data type for model weights and activations. The "auto" option will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.
89
- --enforce-eager enforce eager execution
90
- --kv-cache-dtype {auto,fp8} Data type for kv cache storage. If "auto", will use model data type. FP8_E5M2 (without scaling) is only supported on cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported ```for common inference criteria.
91
- --quantization-param-path QUANT_PARAM_JSON Path to the JSON file containing the KV cache scaling factors. This should generally be supplied, when KV cache dtype is FP8. Otherwise, KV cache scaling factors default to 1.0, which may cause accuracy issues. FP8_E5M2 (without scaling) is only supported on cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for common inference criteria.
92
-```
93
-Example:
94
-```console
95
-python3 benchmarks/benchmark_throughput.py --input-len <INPUT_LEN> --output-len <OUTPUT_LEN> -tp <TENSOR_PARALLEL_SIZE> --kv-cache-dtype fp8 --quantization-param-path <path/to/kv_cache_scales.json> --model <path-to-llama2>
96
-```
examples/other/fp8/extract_scales.py
deleted
-367
@@ -1,367 +0,0 @@
1
-import argparse
2
-import glob
3
-import json
4
-import os
5
-from typing import Any, Callable, Dict, List, Optional, Tuple
6
-
7
-import numpy as np
8
-import torch
9
-from safetensors.torch import safe_open
10
-
11
-from vllm.model_executor.layers.quantization.schema import QuantParamSchema
12
-
13
-
14
-# Adapted from vllm/model_executor/model_loader/weight_utils.py
15
-# The main differences are that we add the NPZ format and simplify
16
-# its functionality drastically for our purposes (e.g. we assume that
17
-# the quantized model exists locally and there is no need to download it)
18
-def _prepare_hf_weights(
19
- quantized_model_dir: str,
20
- load_format: str = "auto",
21
- fall_back_to_pt: bool = True,
22
-) -> Tuple[List[str], bool]:
23
- if not os.path.isdir(quantized_model_dir):
24
- raise FileNotFoundError(
25
- f"The quantized model directory `{quantized_model_dir}` "
26
- "does not exist.")
27
- use_safetensors = False
28
- # Some quantized models use .pt files for storing the weights.
29
- if load_format == "auto":
30
- allow_patterns = ["*.safetensors", "*.bin"]
31
- elif load_format == "safetensors":
32
- use_safetensors = True
33
- allow_patterns = ["*.safetensors"]
34
- elif load_format == "pt":
35
- allow_patterns = ["*.pt"]
36
- elif load_format == "npz":
37
- allow_patterns = ["*.npz"]
38
- else:
39
- raise ValueError(f"Unknown load_format: {load_format}")
40
- if fall_back_to_pt:
41
- allow_patterns += ["*.pt"]
42
-
43
- hf_weights_files: List[str] = []
44
- for pattern in allow_patterns:
45
- hf_weights_files += glob.glob(
46
- os.path.join(quantized_model_dir, pattern))
47
- if len(hf_weights_files) > 0:
48
- if pattern == "*.safetensors":
49
- use_safetensors = True
50
- break
51
-
52
- if not use_safetensors:
53
- # Exclude files that are not needed for inference.
54
- # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
55
- blacklist = [
56
- "training_args.bin",
57
- "optimizer.bin",
58
- "optimizer.pt",
59
- "scheduler.pt",
60
- "scaler.pt",
61
- ]
62
- hf_weights_files = [
63
- f for f in hf_weights_files
64
- if not any(f.endswith(x) for x in blacklist)
65
- ]
66
-
67
- if len(hf_weights_files) == 0:
68
- raise RuntimeError(
69
- f"Cannot find any model weights with `{quantized_model_dir}`")
70
-
71
- return hf_weights_files, use_safetensors
72
-
73
-
74
-# Adapted from vllm/model_executor/model_loader/weight_utils.py
75
-def _hf_tensorfile_iterator(filename: str, load_format: str,
76
- use_safetensors: bool):
77
- if load_format == "npz":
78
- assert not use_safetensors
79
- with np.load(filename) as data:
80
- for name in data.files:
81
- param = torch.from_numpy(data[name])
82
- yield name, param
83
- elif use_safetensors:
84
- with safe_open(filename, framework="pt") as f:
85
- for name in f.keys(): # NOQA: SIM118
86
- param = f.get_tensor(name)
87
- yield name, param
88
- else:
89
- state = torch.load(filename, map_location="cpu")
90
- for name, param in state.items():
91
- yield name, param
92
- del state
93
- torch.cuda.empty_cache()
94
-
95
-
96
-def _kv_scales_extractor(
97
- hf_tensor_files: List[str],
98
- use_safetensors: bool,
99
- rank_keyword: str = "rank",
100
- expected_tp_size: Optional[int] = None) -> Dict[int, Dict[int, float]]:
101
- """
102
- Given a list of files containing tensor data, attempt to extract KV cache
103
- scales from these files. Intended as a helper function taking in the output
104
- from _prepare_hf_weights.
105
- Args:
106
- rank_keyword Matches the number immediately after this keyword in the
107
- tensor filename to determine the TP rank corresponding
108
- to said tensor file
109
- expected_tp_size If specified, the TP size of the tensor files is checked
110
- against this and an error is raised if they don't match.
111
- Returns a dictionary mapping TP ranks to their relevant KV cache scales.
112
- The per-rank scales are themselves represented as a dictionary of layer
113
- indices to the respective per-layer scale.
114
- """
115
- for char in rank_keyword:
116
- assert not char.isdecimal(
117
- ), f"Rank keyword {rank_keyword} contains a numeric character!"
118
- rank_scales_map: Dict[int, Dict[int, float]] = {}
119
- for tensor_file in hf_tensor_files:
120
- try:
121
- rank_idx = tensor_file.find(rank_keyword)
122
- if rank_idx != -1:
123
- start_idx = rank_idx + len(rank_keyword)
124
- stop_idx = start_idx
125
- while stop_idx < len(
126
- tensor_file) and tensor_file[stop_idx].isdecimal():
127
- stop_idx += 1
128
- if stop_idx == start_idx:
129
- raise RuntimeError("Did not find rank # in filename.")
130
- rank = int(tensor_file[start_idx:stop_idx])
131
- elif len(hf_tensor_files) == 1:
132
- # Since there is only one tensor file, we can assume
133
- # that it's intended for TP rank 0
134
- rank = 0
135
- else:
136
- raise RuntimeError(
137
- f"Filename does not contain '{rank_keyword}'.")
138
- except RuntimeError:
139
- print("Unable to determine TP rank "
140
- f"corresponding to file '{tensor_file}'")
141
- raise
142
-
143
- if rank not in rank_scales_map:
144
- layer_scales_map: Dict[int, float] = {}
145
- rank_scales_map[rank] = layer_scales_map
146
- else:
147
- raise RuntimeError(
148
- f"Tensor file '{tensor_file}' shares TP rank {rank} "
149
- "with another tensor file.")
150
-
151
- module_delimiter = ":" if args.load_format == "npz" else "."
152
- for name, param in _hf_tensorfile_iterator(tensor_file,
153
- args.load_format,
154
- use_safetensors):
155
- if "kv_cache_scaling_factor" in name:
156
- nums = [
157
- int(s) for s in name.split(module_delimiter)
158
- if s.isdecimal()
159
- ]
160
- assert len(
161
- nums) == 1, f"Could not determine layer idx for {name}"
162
- layer_idx = nums[0]
163
- assert layer_idx not in layer_scales_map, f"Duplicate scaling"\
164
- f" factor corresponding to layer {layer_idx}"
165
- try:
166
- layer_scales_map[layer_idx] = param.item()
167
- except RuntimeError:
168
- print(
169
- "This utility supports only per-tensor scalar scales "
170
- f"for now. The tensor\n {name} = {param} \nis an "
171
- "invalid scale factor.")
172
- raise
173
-
174
- if all(
175
- len(layer_scales_map) == 0
176
- for layer_scales_map in rank_scales_map.values()):
177
- # Note: this is true even if the rank_scales_map is empty
178
- print("WARNING: No KV cache scale factors found. No output saved.")
179
- return None
180
- empirical_tp_world_size = max(rank_scales_map.keys()) + 1
181
- if expected_tp_size is not None:
182
- assert expected_tp_size == empirical_tp_world_size, \
183
- f"User expected TP world size = {expected_tp_size} " \
184
- "from model but tool is expecting TP world size = " \
185
- f"{empirical_tp_world_size} from model instead."
186
- for i in range(empirical_tp_world_size):
187
- assert i in rank_scales_map, "Expected TP world size = "\
188
- f"{empirical_tp_world_size} but did not find KV " \
189
- f"cache scaling factors for TP rank {i}"
190
- print(f"Found TP world size = {empirical_tp_world_size} "
191
- "when extracting KV cache scales!")
192
- return rank_scales_map
193
-
194
-
195
-def _metadata_extractor(quantized_model_dir: str,
196
- metadata_extract_fns: \
197
- Dict[str, Callable[[Dict[str, Any]], Any]]) \
198
- -> Dict[str, Any]:
199
- """
200
- Given a directory containing quantized model files, this function
201
- aims to extract metadata from the JSON files within this directory.
202
- Each JSON file is expected to represent a dictionary in JSON
203
- format (referred to as a "JSON-dictionary"). Metadata extraction is
204
- defined by a dictionary called metadata_extract_fns, where each
205
- metadata field name is mapped to an extraction function.
206
-
207
- These extraction functions are designed to take a JSON-dictionary
208
- as their only argument and return the corresponding metadata.
209
- While extraction functions are permitted to raise exceptions, they
210
- should only raise a KeyError or ValueError if the metadata field
211
- cannot be extracted from the current JSON-dictionary, yet there's
212
- a possibility of finding it in another JSON-dictionary.
213
-
214
- The function returns a dictionary that maps metadata fields to
215
- their extracted data. The keys of this dictionary correspond exactly
216
- to those in metadata_extract_fns. If any fields fail to be extracted,
217
- their corresponding values are set to None, and a warning is printed.
218
- """
219
- if not os.path.isdir(quantized_model_dir):
220
- raise FileNotFoundError(
221
- f"The quantized model directory `{quantized_model_dir}` "
222
- "does not exist.")
223
- metadata_files = glob.glob(os.path.join(quantized_model_dir, "*.json"))
224
-
225
- result: Dict[str, Any] = {}
226
- for file in metadata_files:
227
- with open(file) as f:
228
- try:
229
- metadata = json.load(f)
230
- except json.JSONDecodeError:
231
- print(f"Could not parse `{file}` as a valid metadata file,"
232
- " skipping it.")
233
- continue
234
- if not isinstance(metadata, dict):
235
- print(f"The file `{file}` does not correspond to a "
236
- "JSON-serialized dictionary, skipping it.")
237
- continue
238
- for metadata_name, extract_fn in metadata_extract_fns.items():
239
- try:
240
- metadata_info = extract_fn(metadata)
241
- if metadata_name not in result:
242
- result[metadata_name] = metadata_info
243
- elif metadata_info != result[metadata_name]:
244
- raise RuntimeError(
245
- "Metadata mismatch! Originally found "
246
- f"{metadata_name} = {result[metadata_name]} but "
247
- f"now found {metadata_name} = {metadata_info} in "
248
- f"`{file}`")
249
- except KeyError:
250
- # It is possible that a given file does not contain some
251
- # of our selected metadata as it could be located in some
252
- # other metadata file.
253
- # 'EFINAE': extract_fn failure is not an error.
254
- pass
255
- except ValueError:
256
- # See above.
257
- pass
258
-
259
- # Warn if we cannot find any of the requested metadata
260
- for metadata_name in metadata_extract_fns:
261
- if metadata_name not in result:
262
- print("WARNING: Unable to find requested metadata field "
263
- f"`{metadata_name}`, setting it to None.")
264
- result[metadata_name] = None
265
-
266
- return result
267
-
268
-
269
-def main(args):
270
- metadata_extract_fns = {
271
- "model_type": lambda json_dict: json_dict["layers"][0]["decoder_type"],
272
- "tp_size": lambda json_dict: int(json_dict["tensor_parallel"]),
273
- "model_dtype": lambda json_dict: json_dict["dtype"]
274
- }
275
- recovered_metadata = _metadata_extractor(args.quantized_model,
276
- metadata_extract_fns)
277
- if args.tp_size is not None:
278
- metadata_tp_size = recovered_metadata["tp_size"]
279
- if metadata_tp_size is not None:
280
- assert args.tp_size == metadata_tp_size, \
281
- f"User expected TP world size = {args.tp_size} " \
282
- f"but found TP world size = {metadata_tp_size} from metadata!"
283
- expected_tp_size = args.tp_size or recovered_metadata["tp_size"]
284
- rank_keyword = "rank"
285
- hf_tensor_files, use_safetensors = _prepare_hf_weights(
286
- args.quantized_model, args.load_format)
287
- rank_scales_map = _kv_scales_extractor(hf_tensor_files, use_safetensors,
288
- rank_keyword, expected_tp_size)
289
- # Postprocess: formatting to the current schema. Consider pulling it
290
- # out into a dedicated function should it ever become more complicated.
291
- rank_scales_map = {
292
- rank: {k: scale[k]
293
- for k in sorted(scale.keys())}
294
- for rank, scale in rank_scales_map.items()
295
- }
296
- # TODO: Expand this with activation and weights scaling factors when
297
- # they are used in the future
298
- schema = QuantParamSchema(
299
- model_type=recovered_metadata["model_type"],
300
- kv_cache={
301
- "dtype": ("float8_e4m3fn" if len(rank_scales_map) > 0 else
302
- recovered_metadata["model_dtype"]),
303
- "scaling_factor":
304
- rank_scales_map
305
- },
306
- )
307
-
308
- if args.output_dir is None:
309
- output_file = os.path.join(args.quantized_model, args.output_name)
310
- else:
311
- if not os.path.isdir(args.output_dir):
312
- os.makedirs(args.output_dir, exist_ok=True)
313
- output_file = os.path.join(args.output_dir, args.output_name)
314
-
315
- with open(output_file, 'w') as f:
316
- f.write(schema.model_dump_json(indent=4))
317
- print(f"Completed! KV cache scaling factors saved to {output_file}")
318
-
319
-
320
-if __name__ == "__main__":
321
- parser = argparse.ArgumentParser(
322
- description="This simple utility extracts the "
323
- "KV cache scaling factors from a quantized HF model "
324
- "and saves them to a JSON file compatible with later "
325
- "use by vLLM (pass this file to the appropriate "
326
- "runtime typically using the argument "
327
- "--quantization-param-path <filename>). This is only used "
328
- "if the KV cache dtype is FP8 and on ROCm (AMD GPU).")
329
- parser.add_argument(
330
- "--quantized-model",
331
- help="Specify the directory containing a single quantized HF model. "
332
- "It is expected that the quantization format is FP8_E4M3, for use "
333
- "on ROCm (AMD GPU).",
334
- required=True)
335
- parser.add_argument(
336
- "--load_format",
337
- help="Optionally specify the format of the model's tensor files "
338
- "containing the KV cache scaling factors.",
339
- choices=["auto", "safetensors", "npz", "pt"],
340
- default="auto")
341
- parser.add_argument(
342
- "--output-dir",
343
- help="Optionally specify the output directory. By default the "
344
- "KV cache scaling factors will be saved in the model directory, "
345
- "however you can override this behavior here.",
346
- default=None)
347
- parser.add_argument(
348
- "--output-name",
349
- help="Optionally specify the output filename.",
350
- # TODO: Change this once additional scaling factors are enabled
351
- default="kv_cache_scales.json")
352
- parser.add_argument(
353
- "--tp-size",
354
- help="Optionally specify the tensor-parallel (TP) size that the "
355
- "quantized model should correspond to. If specified, during KV "
356
- "cache scaling factor extraction the observed TP size will be "
357
- "checked against this and an error will be raised if there is "
358
- "a mismatch. If not specified, the quantized model's expected "
359
- "TP size is instead inferred from the largest TP rank observed. "
360
- "The expected TP size is cross-checked against the TP ranks "
361
- "observed in the quantized model and an error is raised if any "
362
- "discrepancies are found.",
363
- default=None,
364
- type=int)
365
- args = parser.parse_args()
366
-
367
- main(args)
examples/other/fp8/quantizer/README.md
deleted
-32
@@ -1,32 +0,0 @@
1
-### Quantizer Utilities
2
-`quantize.py`: NVIDIA Quantization utilities using TensorRT-Model-Optimizer, ported
3
-from TensorRT-LLM: [`examples/quantization/quantize.py`](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/quantization/quantize.py)
4
-
5
-### Prerequisite
6
-
7
-#### AMMO (AlgorithMic Model Optimization) Installation: nvidia-ammo 0.7.1 or later
8
-`pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com nvidia-ammo`
9
-
10
-#### AMMO Download (code and docs)
11
-`https://developer.nvidia.com/downloads/assets/cuda/files/nvidia-ammo/nvidia_ammo-0.5.0.tar.gz`
12
-`https://developer.nvidia.com/downloads/assets/cuda/files/nvidia-ammo/nvidia_ammo-0.7.1.tar.gz`
13
-
14
-### Usage
15
-
16
-#### Run on H100 system for speed if FP8; number of GPUs depends on the model size
17
-
18
-#### Example: quantize Llama2-7b model from HF to FP8 with FP8 KV Cache:
19
-`python quantize.py --model-dir ./ll2-7b --dtype float16 --qformat fp8 --kv-cache-dtype fp8 --output-dir ./ll2_7b_fp8 --calib-size 512 --tp-size 1`
20
-
21
-Outputs: model structure, quantized model & parameters (with scaling factors) are in JSON and Safetensors (npz is generated only for the reference)
22
-```
23
-# ll ./ll2_7b_fp8/
24
-total 19998244
25
-drwxr-xr-x 2 root root 4096 Feb 7 01:08 ./
26
-drwxrwxr-x 8 1060 1061 4096 Feb 7 01:08 ../
27
--rw-r--r-- 1 root root 176411 Feb 7 01:08 llama_tp1.json
28
--rw-r--r-- 1 root root 13477087480 Feb 7 01:09 llama_tp1_rank0.npz
29
--rw-r--r-- 1 root root 7000893272 Feb 7 01:08 rank0.safetensors
30
-#
31
-```
32
-
examples/other/fp8/quantizer/quantize.py
deleted
-367
@@ -1,367 +0,0 @@
1
-# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # noqa: E501
2
-# SPDX-License-Identifier: Apache-2.0
3
-#
4
-# Licensed under the Apache License, Version 2.0 (the "License");
5
-# you may not use this file except in compliance with the License.
6
-# You may obtain a copy of the License at
7
-#
8
-# http://www.apache.org/licenses/LICENSE-2.0
9
-#
10
-# Unless required by applicable law or agreed to in writing, software
11
-# distributed under the License is distributed on an "AS IS" BASIS,
12
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
-# See the License for the specific language governing permissions and
14
-# limitations under the License.
15
-"""
16
-Adapted from examples/quantization/hf_ptq.py
17
-"""
18
-
19
-import argparse
20
-import copy
21
-import json
22
-import random
23
-import time
24
-
25
-import ammo.torch.quantization as atq
26
-import numpy as np
27
-import torch
28
-from ammo.torch.export import export_model_config
29
-from datasets import load_dataset
30
-from torch.utils.data import DataLoader
31
-from transformers import AutoModelForCausalLM, AutoTokenizer
32
-
33
-RAND_SEED = 1234
34
-MAX_SEQ_LEN = 2048
35
-
36
-EMPTY_CFG = {
37
- "quant_cfg": {
38
- "*weight_quantizer": {
39
- "enable": False,
40
- },
41
- "*input_quantizer": {
42
- "enable": False
43
- },
44
- "*lm_head*": {
45
- "enable": False
46
- },
47
- "*output_layer*": {
48
- "enable": False
49
- },
50
- "default": {
51
- "enable": False
52
- },
53
- },
54
- "algorithm": "max",
55
-}
56
-
57
-KV_CACHE_CFG = {
58
- "*.query_key_value.output_quantizer": {
59
- "num_bits": 8,
60
- "axis": None,
61
- "enable": True
62
- },
63
- "*.Wqkv.output_quantizer": {
64
- "num_bits": 8,
65
- "axis": None,
66
- "enable": True
67
- },
68
- "*.W_pack.output_quantizer": {
69
- "num_bits": 8,
70
- "axis": None,
71
- "enable": True
72
- },
73
- "*.c_attn.output_quantizer": {
74
- "num_bits": 8,
75
- "axis": None,
76
- "enable": True
77
- },
78
- "*.k_proj.output_quantizer": {
79
- "num_bits": 8,
80
- "axis": None,
81
- "enable": True
82
- },
83
- "*.v_proj.output_quantizer": {
84
- "num_bits": 8,
85
- "axis": None,
86
- "enable": True
87
- },
88
-}
89
-
90
-QUANT_CFG_CHOICES = {
91
- "int8_sq": atq.INT8_SMOOTHQUANT_CFG,
92
- "fp8": atq.FP8_DEFAULT_CFG,
93
- "int4_awq": atq.INT4_AWQ_CFG,
94
- "w4a8_awq": atq.W4A8_AWQ_BETA_CFG,
95
- "int8_wo": EMPTY_CFG,
96
- "int4_wo": EMPTY_CFG,
97
- "full_prec": EMPTY_CFG,
98
-}
99
-
100
-MODEL_NAME_PATTERN_MAP = {
101
- "GPT2": "gpt2",
102
- "Xverse": "llama",
103
- "Llama": "llama",
104
- "Mistral": "llama",
105
- "GPTJ": "gptj",
106
- "FalconForCausalLM": "falcon",
107
- "RWForCausalLM": "falcon",
108
- "baichuan": "baichuan",
109
- "MPT": "mpt",
110
- "Bloom": "bloom",
111
- "ChatGLM": "chatglm",
112
- "QWen": "qwen",
113
-}
114
-
115
-
116
-def get_tokenizer(ckpt_path, max_seq_len=MAX_SEQ_LEN, model_type=None):
117
- print(f"Initializing tokenizer from {ckpt_path}")
118
- tokenizer = AutoTokenizer.from_pretrained(
119
- ckpt_path,
120
- model_max_length=max_seq_len,
121
- padding_side="left",
122
- trust_remote_code=True,
123
- )
124
- if model_type and model_type == "qwen":
125
- # qwen use token id 151643 as pad and eos tokens
126
- tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
127
- tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
128
-
129
- # can't set attribute 'pad_token' for "<unk>"
130
- if tokenizer.pad_token != "<unk>":
131
- tokenizer.pad_token = tokenizer.eos_token
132
- if tokenizer.pad_token is None:
133
- tokenizer.pad_token = tokenizer.eos_token
134
- assert (tokenizer.pad_token
135
- is not None), f"Pad token for {model_type} cannot be set!"
136
-
137
- return tokenizer
138
-
139
-
140
-def get_model(ckpt_path, dtype="fp16", device="cuda"):
141
- print(f"Initializing model from {ckpt_path}")
142
- if dtype == "bf16" or dtype == "bfloat16":
143
- dtype = torch.bfloat16
144
- elif dtype == "fp16" or dtype == "float16":
145
- dtype = torch.float16
146
- elif dtype == "fp32" or dtype == "float32":
147
- dtype = torch.float32
148
- else:
149
- raise NotImplementedError(f"Unknown dtype {dtype}")
150
-
151
- # model_kwargs = {"torch_dtype": dtype}
152
- model_kwargs = {"torch_dtype": "auto"}
153
-
154
- model = AutoModelForCausalLM.from_pretrained(ckpt_path,
155
- device_map="auto",
156
- **model_kwargs,
157
- trust_remote_code=True)
158
- model.eval()
159
-
160
- model_dtype = next(model.parameters()).dtype
161
- if dtype != model_dtype:
162
- print("[TensorRT-LLM][WARNING] The manually set model data type is "
163
- f"{dtype}, but the data type of the HuggingFace model is "
164
- f"{model_dtype}.")
165
-
166
- return model
167
-
168
-
169
-def get_model_type(model):
170
- for k, v in MODEL_NAME_PATTERN_MAP.items():
171
- if k.lower() in type(model).__name__.lower():
172
- return v
173
- return None
174
-
175
-
176
-def get_calib_dataloader(data="cnn_dailymail",
177
- tokenizer=None,
178
- batch_size=1,
179
- calib_size=512,
180
- block_size=512,
181
- device=None):
182
- print("Loading calibration dataset")
183
- if data == "pileval":
184
- dataset = load_dataset(
185
- "json",
186
- data_files="https://the-eye.eu/public/AI/pile/val.jsonl.zst",
187
- split="train")
188
- dataset = dataset["text"][:calib_size]
189
- elif data == "cnn_dailymail":
190
- dataset = load_dataset("cnn_dailymail", name="3.0.0", split="train")
191
- dataset = dataset["article"][:calib_size]
192
- else:
193
- raise NotImplementedError
194
-
195
- batch_encoded = tokenizer.batch_encode_plus(dataset,
196
- return_tensors="pt",
197
- padding="max_length",
198
- truncation=True,
199
- max_length=block_size)
200
- if device:
201
- batch_encoded = batch_encoded.to(device)
202
- batch_encoded = batch_encoded["input_ids"]
203
-
204
- calib_dataloader = DataLoader(batch_encoded,
205
- batch_size=batch_size,
206
- shuffle=False)
207
-
208
- return calib_dataloader
209
-
210
-
211
-def quantize_model(model, quant_cfg, calib_dataloader=None):
212
-
213
- def calibrate_loop():
214
- if calib_dataloader is None:
215
- return
216
- """Adjusts weights and scaling factors based on selected algorithms."""
217
- for idx, data in enumerate(calib_dataloader):
218
- print(f"Calibrating batch {idx}")
219
- model(data)
220
-
221
- print("Starting quantization...")
222
- start_time = time.time()
223
- atq.quantize(model, quant_cfg, forward_loop=calibrate_loop)
224
- end_time = time.time()
225
- print("Quantization done. Total time used: {:.2f} s.".format(end_time -
226
- start_time))
227
-
228
- return model
229
-
230
-
231
-def main(args):
232
- if not torch.cuda.is_available():
233
- raise OSError("GPU is required for inference.")
234
-
235
- random.seed(RAND_SEED)
236
- np.random.seed(RAND_SEED)
237
-
238
- model = get_model(args.model_dir, args.dtype, args.device)
239
- model_type = get_model_type(model)
240
- tokenizer = get_tokenizer(args.model_dir, model_type=model_type)
241
-
242
- if args.qformat in ["full_prec", "int8_wo", "int4_wo"
243
- ] and args.kv_cache_dtype is None:
244
- print(f"No quantization applied, export {args.dtype} model")
245
- else:
246
- if "awq" in args.qformat:
247
- if args.calib_size > 32:
248
- print("AWQ calibration could take longer with calib_size = "
249
- f"{args.calib_size}, Using calib_size=32 instead")
250
- args.calib_size = 32
251
- print("\nAWQ calibration could take longer than other calibration "
252
- "methods. Please increase the batch size to speed up the "
253
- "calibration process. Batch size can be set by adding the "
254
- "argument --batch_size <batch_size> to the command line.\n")
255
-
256
- calib_dataloader = get_calib_dataloader(
257
- tokenizer=tokenizer,
258
- batch_size=args.batch_size,
259
- calib_size=args.calib_size,
260
- device=args.device,
261
- )
262
-
263
- if args.qformat in QUANT_CFG_CHOICES:
264
- quant_cfg = QUANT_CFG_CHOICES[args.qformat]
265
- else:
266
- raise ValueError(
267
- f"Unsupported quantization format: {args.qformat}")
268
-
269
- if "awq" in args.qformat:
270
- quant_cfg = copy.deepcopy(QUANT_CFG_CHOICES[args.qformat])
271
- weight_quantizer = quant_cfg["quant_cfg"][
272
- "*weight_quantizer"] # type: ignore
273
- if isinstance(weight_quantizer, list):
274
- weight_quantizer = weight_quantizer[0]
275
- weight_quantizer["block_sizes"][-1] = args.awq_block_size
276
-
277
- if args.kv_cache_dtype is not None:
278
- if args.kv_cache_dtype == "fp8":
279
- for value in KV_CACHE_CFG.values():
280
- value.update({"num_bits": (4, 3)}) # type: ignore
281
- quant_cfg["quant_cfg"].update(KV_CACHE_CFG) # type: ignore
282
-
283
- print(quant_cfg)
284
-
285
- model = quantize_model(model, quant_cfg, calib_dataloader)
286
-
287
- with torch.inference_mode():
288
- if model_type is None:
289
- print(f"Unknown model type {type(model).__name__}. Continue "
290
- "exporting...")
291
- model_type = f"unknown:{type(model).__name__}"
292
-
293
- export_path = args.output_dir
294
- start_time = time.time()
295
-
296
- if args.qformat == "int4_awq" and model_type == "qwen":
297
- torch.save(model.state_dict(), export_path)
298
- else:
299
- export_npz = (model_type not in [
300
- 'gptj', 'falcon', 'chatglm', 'mpt', 'llama', 'baichuan'
301
- ])
302
-
303
- # export safetensors
304
- export_model_config(
305
- model,
306
- model_type,
307
- getattr(torch, args.dtype),
308
- export_dir=export_path,
309
- inference_tensor_parallel=args.tp_size,
310
- inference_pipeline_parallel=args.pp_size,
311
- # export_tensorrt_llm_config=(not export_npz),
312
- export_tensorrt_llm_config=False,
313
- export_npz=export_npz)
314
-
315
- # Workaround for wo quantization
316
- if args.qformat in ["int8_wo", "int4_wo", "full_prec"]:
317
- with open(f"{export_path}/config.json") as f:
318
- tensorrt_llm_config = json.load(f)
319
- if args.qformat == "int8_wo":
320
- tensorrt_llm_config["quantization"]["quant_algo"] = 'W8A16'
321
- elif args.qformat == "int4_wo":
322
- tensorrt_llm_config["quantization"]["quant_algo"] = 'W4A16'
323
- else:
324
- tensorrt_llm_config["quantization"]["quant_algo"] = None
325
- with open(f"{export_path}/config.json", "w") as f:
326
- json.dump(tensorrt_llm_config, f, indent=4)
327
-
328
- end_time = time.time()
329
- print("Quantized model exported to {} \nTotal time used {:.2f} s.".
330
- format(export_path, end_time - start_time))
331
-
332
-
333
-if __name__ == "__main__":
334
- parser = argparse.ArgumentParser(description=__doc__)
335
- parser.add_argument("--model-dir",
336
- help="Specify where the HuggingFace model is",
337
- required=True)
338
- parser.add_argument("--device", default="cuda")
339
- parser.add_argument("--dtype", help="Model data type.", default="float16")
340
- parser.add_argument(
341
- "--qformat",
342
- help="Quantization format.",
343
- default="full_prec",
344
- choices=[
345
- "fp8", "int8_sq", "int4_awq", "w4a8_awq", "int8_wo", "int4_wo",
346
- "full_prec"
347
- ],
348
- )
349
- parser.add_argument("--batch-size",
350
- help="Batch size for calibration.",
351
- type=int,
352
- default=1)
353
- parser.add_argument("--calib-size",
354
- help="Number of samples for calibration.",
355
- type=int,
356
- default=512)
357
- parser.add_argument("--output-dir", default="exported_model")
358
- parser.add_argument("--tp-size", type=int, default=1)
359
- parser.add_argument("--pp-size", type=int, default=1)
360
- parser.add_argument("--awq-block-size", type=int, default=128)
361
- parser.add_argument("--kv-cache-dtype",
362
- help="KV Cache dtype.",
363
- default=None,
364
- choices=["int8", "fp8", None])
365
- args = parser.parse_args()
366
-
367
- main(args)
tests/fp8_kv/llama2-70b-fp8-kv/kv_cache_scales.json
deleted
-90
@@ -1,90 +0,0 @@
1
-{
2
- "model_type": "llama",
3
- "kv_cache": {
4
- "dtype": "float8_e4m3fn",
5
- "scaling_factor": {
6
- "0": {
7
- "0": 0.0230364128947258,
8
- "1": 0.01979283057153225,
9
- "2": 0.0241350457072258,
10
- "3": 0.0308314748108387,
11
- "4": 0.0430733822286129,
12
- "5": 0.0370396226644516,
13
- "6": 0.0306222103536129,
14
- "7": 0.0357491634786129,
15
- "8": 0.0358189195394516,
16
- "9": 0.0443289652466774,
17
- "10": 0.0433175228536129,
18
- "11": 0.0416782945394516,
19
- "12": 0.0366908498108387,
20
- "13": 0.0432477705180645,
21
- "14": 0.0410505048930645,
22
- "15": 0.0457589291036129,
23
- "16": 0.0418526791036129,
24
- "17": 0.0432477705180645,
25
- "18": 0.0469447560608387,
26
- "19": 0.0514787957072258,
27
- "20": 0.0541294664144516,
28
- "21": 0.0587681382894516,
29
- "22": 0.0625,
30
- "23": 0.0585588738322258,
31
- "24": 0.0600237175822258,
32
- "25": 0.0588030144572258,
33
- "26": 0.0531180277466774,
34
- "27": 0.06396484375,
35
- "28": 0.0603027381002903,
36
- "29": 0.0582101047039032,
37
- "30": 0.0625348836183548,
38
- "31": 0.0585588738322258,
39
- "32": 0.0582798570394516,
40
- "33": 0.0575125589966774,
41
- "34": 0.0590820349752903,
42
- "35": 0.0614188089966774,
43
- "36": 0.0631975457072258,
44
- "37": 0.0615931935608387,
45
- "38": 0.0601283498108387,
46
- "39": 0.0571986623108387,
47
- "40": 0.0670340433716774,
48
- "41": 0.0523507259786129,
49
- "42": 0.0547223798930645,
50
- "43": 0.0631975457072258,
51
- "44": 0.0663713738322258,
52
- "45": 0.0603376142680645,
53
- "46": 0.0652204304933548,
54
- "47": 0.0734514519572258,
55
- "48": 0.0693708211183548,
56
- "49": 0.0725446492433548,
57
- "50": 0.0627790242433548,
58
- "51": 0.0691266804933548,
59
- "52": 0.0688825398683548,
60
- "53": 0.068429134786129,
61
- "54": 0.0605119988322258,
62
- "55": 0.0799386203289032,
63
- "56": 0.0853097140789032,
64
- "57": 0.0661969929933548,
65
- "58": 0.0689871683716774,
66
- "59": 0.0724051371216774,
67
- "60": 0.0541643425822258,
68
- "61": 0.0626743882894516,
69
- "62": 0.0628487765789032,
70
- "63": 0.0607212632894516,
71
- "64": 0.0589076466858387,
72
- "65": 0.0451660193502903,
73
- "66": 0.0453055277466774,
74
- "67": 0.0414341539144516,
75
- "68": 0.0385044664144516,
76
- "69": 0.0414341539144516,
77
- "70": 0.0466308631002903,
78
- "71": 0.0399693101644516,
79
- "72": 0.0437011756002903,
80
- "73": 0.0434221550822258,
81
- "74": 0.0428989976644516,
82
- "75": 0.0401785746216774,
83
- "76": 0.0431082621216774,
84
- "77": 0.0484444759786129,
85
- "78": 0.0417829267680645,
86
- "79": 0.0418178029358387
87
- }
88
- }
89
- }
90
-}
\ No newline at end of file
tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json
deleted
-42
@@ -1,42 +0,0 @@
1
-{
2
- "model_type": "llama",
3
- "kv_cache": {
4
- "dtype": "float8_e4m3fn",
5
- "scaling_factor": {
6
- "0": {
7
- "0": 0.0152239128947258,
8
- "1": 0.0188860222697258,
9
- "2": 0.0354178324341774,
10
- "3": 0.0376674123108387,
11
- "4": 0.0418526791036129,
12
- "5": 0.0433175228536129,
13
- "6": 0.0397600457072258,
14
- "7": 0.0424455925822258,
15
- "8": 0.0415387861430645,
16
- "9": 0.0408412404358387,
17
- "10": 0.0395856611430645,
18
- "11": 0.0377371683716774,
19
- "12": 0.0400739423930645,
20
- "13": 0.040771484375,
21
- "14": 0.0393415205180645,
22
- "15": 0.0369001142680645,
23
- "16": 0.03857421875,
24
- "17": 0.0387486070394516,
25
- "18": 0.0403180830180645,
26
- "19": 0.0396205373108387,
27
- "20": 0.0375627800822258,
28
- "21": 0.0407366082072258,
29
- "22": 0.0432477705180645,
30
- "23": 0.0377022884786129,
31
- "24": 0.0399693101644516,
32
- "25": 0.0374581478536129,
33
- "26": 0.0413295216858387,
34
- "27": 0.0442243330180645,
35
- "28": 0.0424804724752903,
36
- "29": 0.0456891767680645,
37
- "30": 0.0409109964966774,
38
- "31": 0.0482352152466774
39
- }
40
- }
41
- }
42
-}
tests/kernels/test_attention.py
+1
-1
@@ -182,7 +182,7 @@ def test_paged_attention(
182
key_cache, value_cache = key_caches[0], value_caches[0]
183
184
# Using default kv_scale
185
- k_scale = v_scale = 1.0
185
+ k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
186
187
# Call the paged attention kernel.
188
output = torch.empty_like(query)
tests/kernels/test_blocksparse_attention.py
+1
-1
@@ -210,7 +210,7 @@ def test_paged_attention(
210
key_cache, value_cache = key_caches[0], value_caches[0]
211
212
# Using default kv_scale
213
- k_scale = v_scale = 1.0
213
+ k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
214
tp_rank = 0
215
216
# Call the paged attention kernel.
tests/kernels/test_cache.py
+5
-5
@@ -160,7 +160,7 @@ def test_reshape_and_cache(
160
cloned_value_cache = value_cache.clone()
161
162
# Using default kv_scale
163
- k_scale = v_scale = 1.0
163
+ k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
164
165
# Call the reshape_and_cache kernel.
166
opcheck(torch.ops._C_cache_ops.reshape_and_cache,
@@ -258,8 +258,8 @@ def test_reshape_and_cache_flash(
258
del key_caches
259
del value_caches
260
261
- k_scale = key.amax().item() / 256
262
- v_scale = value.amax().item() / 256
261
+ k_scale = (key.amax() / 256.0).to(torch.float32)
262
+ v_scale = (value.amax() / 256.0).to(torch.float32)
263
264
# Clone the KV caches.
265
if kv_cache_dtype == "fp8":
@@ -284,12 +284,12 @@ def test_reshape_and_cache_flash(
284
result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
285
ops.convert_fp8(result_key_cache,
286
key_cache,
287
- k_scale,
287
+ k_scale.item(),
288
kv_dtype=kv_cache_dtype)
289
result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
290
ops.convert_fp8(result_value_cache,
291
value_cache,
292
- v_scale,
292
+ v_scale.item(),
293
kv_dtype=kv_cache_dtype)
294
295
# Run the reference implementation.
tests/kernels/test_prefix_prefill.py
+10
@@ -138,6 +138,7 @@ def test_contexted_kv_attention(
138
# to V_cache[num_blocks, num_kv_heads, head_size, block_size]
139
v_cache = v_cache.view(-1, block_size, num_kv_heads,
140
head_size).permute(0, 2, 3, 1).contiguous()
141
+ k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
142
143
# Warm up the Triton kernel by calling it once before actually measuring
144
# generation time
@@ -153,6 +154,8 @@ def test_contexted_kv_attention(
154
b_seq_len,
155
b_ctx_len,
156
max_input_len,
157
+ k_scale,
158
+ v_scale,
159
sliding_window=sliding_window)
160
torch.cuda.synchronize()
161
start_time = time.time()
@@ -168,6 +171,8 @@ def test_contexted_kv_attention(
171
b_seq_len,
172
b_ctx_len,
173
max_input_len,
174
+ k_scale,
175
+ v_scale,
176
sliding_window=sliding_window)
177
torch.cuda.synchronize()
178
end_time = time.time()
@@ -366,6 +371,7 @@ def test_contexted_kv_attention_alibi(
371
# to V_cache[num_blocks, num_kv_heads, head_size, block_size]
372
v_cache = v_cache.view(-1, block_size, num_kv_heads,
373
head_size).permute(0, 2, 3, 1).contiguous()
374
+ k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
375
376
# Warm up the Triton kernel by calling it once before actually measuring
377
# generation time
@@ -381,6 +387,8 @@ def test_contexted_kv_attention_alibi(
387
b_seq_len,
388
b_ctx_len,
389
max_input_len,
390
+ k_scale,
391
+ v_scale,
392
alibi_slopes=alibi_slopes)
393
torch.cuda.synchronize()
394
start_time = time.time()
@@ -396,6 +404,8 @@ def test_contexted_kv_attention_alibi(
404
b_seq_len,
405
b_ctx_len,
406
max_input_len,
407
+ k_scale,
408
+ v_scale,
409
alibi_slopes=alibi_slopes)
410
torch.cuda.synchronize()
411
end_time = time.time()
tests/kernels/utils.py
+2
@@ -909,6 +909,7 @@ def make_test_metadata(
909
num_prefills=num_prefills,
910
slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
911
multi_modal_placeholder_index_maps=None,
912
+ enable_kv_scales_calculation=True,
913
num_prefill_tokens=num_prefill_tokens,
914
num_decode_tokens=num_decode_tokens,
915
seq_lens=seq_lens,
@@ -958,6 +959,7 @@ def make_test_metadata(
959
num_prefills=num_prefills,
960
slot_mapping=kv_mmap.slot_mapping,
961
multi_modal_placeholder_index_maps=None,
962
+ enable_kv_scales_calculation=True,
963
num_prefill_tokens=num_prefill_tokens,
964
num_decode_tokens=num_decode_tokens,
965
seq_lens=seq_lens,
tests/models/decoder_only/language/test_fp8.py
+4
-11
@@ -19,18 +19,17 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
19
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
20
reason="fp8 is not supported on this GPU type.")
21
@pytest.mark.parametrize(
22
- "kv_cache_dtype,base_model,test_model,scale_path",
22
+ "kv_cache_dtype,base_model,test_model",
23
[
24
# Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
25
("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
26
- "nm-testing/Llama-3.2-1B-Instruct-FP8-KV", None),
26
+ "nm-testing/Llama-3.2-1B-Instruct-FP8-KV"),
27
# Test FP16 checkpoint w. fp8_e5m2 kv-cache.
28
("fp8_e5m2", "meta-llama/Llama-3.2-1B-Instruct",
29
- "meta-llama/Llama-3.2-1B-Instruct", None),
29
+ "meta-llama/Llama-3.2-1B-Instruct"),
30
# Test FP16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
31
("fp8_e4m3", "meta-llama/Llama-2-7b-chat-hf",
32
- "meta-llama/Llama-2-7b-chat-hf",
33
- "./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json")
32
+ "meta-llama/Llama-2-7b-chat-hf")
33
])
34
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
35
@pytest.mark.parametrize("max_tokens", [4])
@@ -48,7 +47,6 @@ def test_models(
47
kv_cache_dtype: str,
48
base_model: str,
49
test_model: str,
51
- scale_path: Optional[str],
50
max_tokens: int,
51
enforce_eager: bool,
52
backend: str,
@@ -76,10 +74,6 @@ def test_models(
74
baseline_outputs = vllm_model.generate_greedy_logprobs(
75
example_prompts, max_tokens, NUM_LOG_PROBS)
76
79
- extra_kwargs = {}
80
- if scale_path is not None:
81
- extra_kwargs["quantization_param_path"] = scale_path
82
-
77
with vllm_runner(
78
test_model,
79
max_model_len=MAX_MODEL_LEN,
@@ -87,7 +81,6 @@ def test_models(
81
enforce_eager=enforce_eager,
82
kv_cache_dtype=kv_cache_dtype,
83
disable_async_output_proc=disable_async_output_proc,
90
- **extra_kwargs,
84
) as vllm_model:
85
test_outputs = vllm_model.generate_greedy_logprobs(
86
example_prompts, max_tokens, NUM_LOG_PROBS)
tests/worker/test_model_input.py
+3
@@ -74,6 +74,7 @@ def test_model_runner_input():
74
num_decode_tokens=3,
75
slot_mapping=torch.zeros(1),
76
multi_modal_placeholder_index_maps=None,
77
+ enable_kv_scales_calculation=True,
78
)
79
model_input = ModelInputForGPUWithSamplingMetadata(
80
input_tokens=torch.ones(10),
@@ -126,6 +127,7 @@ def test_embedding_model_runner_input():
127
num_decode_tokens=3,
128
slot_mapping=torch.zeros(1),
129
multi_modal_placeholder_index_maps=None,
130
+ enable_kv_scales_calculation=True,
131
)
132
model_input = ModelInputForGPUWithPoolingMetadata(
133
input_tokens=torch.ones(10),
@@ -177,6 +179,7 @@ def test_multi_step_model_runner_input():
179
num_decode_tokens=3,
180
slot_mapping=torch.zeros(1),
181
multi_modal_placeholder_index_maps=None,
182
+ enable_kv_scales_calculation=True,
183
)
184
frozen_model_input = ModelInputForGPUWithSamplingMetadata(
185
input_tokens=torch.ones(10),
vllm/_custom_ops.py
+10
-10
@@ -48,8 +48,8 @@ def paged_attention_v1(
48
max_seq_len: int,
49
alibi_slopes: Optional[torch.Tensor],
50
kv_cache_dtype: str,
51
- k_scale: float,
52
- v_scale: float,
51
+ k_scale: torch.Tensor,
52
+ v_scale: torch.Tensor,
53
tp_rank: int = 0,
54
blocksparse_local_blocks: int = 0,
55
blocksparse_vert_stride: int = 0,
@@ -80,8 +80,8 @@ def paged_attention_v2(
80
max_seq_len: int,
81
alibi_slopes: Optional[torch.Tensor],
82
kv_cache_dtype: str,
83
- k_scale: float,
84
- v_scale: float,
83
+ k_scale: torch.Tensor,
84
+ v_scale: torch.Tensor,
85
tp_rank: int = 0,
86
blocksparse_local_blocks: int = 0,
87
blocksparse_vert_stride: int = 0,
@@ -112,8 +112,8 @@ def paged_attention_rocm(
112
max_seq_len: int,
113
alibi_slopes: Optional[torch.Tensor],
114
kv_cache_dtype: str,
115
- k_scale: float,
116
- v_scale: float,
115
+ k_scale: torch.Tensor,
116
+ v_scale: torch.Tensor,
117
) -> None:
118
torch.ops._rocm_C.paged_attention(out, exp_sum, max_logits, tmp_out, query,
119
key_cache, value_cache, num_kv_heads,
@@ -956,8 +956,8 @@ def reshape_and_cache(
956
value_cache: torch.Tensor,
957
slot_mapping: torch.Tensor,
958
kv_cache_dtype: str,
959
- k_scale: float,
960
- v_scale: float,
959
+ k_scale: torch.Tensor,
960
+ v_scale: torch.Tensor,
961
) -> None:
962
torch.ops._C_cache_ops.reshape_and_cache(key, value, key_cache,
963
value_cache, slot_mapping,
@@ -971,8 +971,8 @@ def reshape_and_cache_flash(
971
value_cache: torch.Tensor,
972
slot_mapping: torch.Tensor,
973
kv_cache_dtype: str,
974
- k_scale: float,
975
- v_scale: float,
974
+ k_scale: torch.Tensor,
975
+ v_scale: torch.Tensor,
976
) -> None:
977
torch.ops._C_cache_ops.reshape_and_cache_flash(key, value, key_cache,
978
value_cache, slot_mapping,
vllm/attention/backends/abstract.py
+8
-2
@@ -123,6 +123,10 @@ class AttentionMetadata:
123
multi_modal_placeholder_index_maps: Optional[Dict[
124
str, MultiModalPlaceholderMap.IndexMap]]
125
126
+ # Enable/disable KV scales calculation. This is so that we can disable the
127
+ # calculation until after prefill and cuda graph capture.
128
+ enable_kv_scales_calculation: bool
129
+
130
@property
131
@abstractmethod
132
def prefill_metadata(self) -> Optional["AttentionMetadata"]:
@@ -226,8 +230,10 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
230
231
class AttentionLayer(Protocol):
232
229
- _k_scale: float
230
- _v_scale: float
233
+ _k_scale: torch.Tensor
234
+ _v_scale: torch.Tensor
235
+ _k_scale_float: float
236
+ _v_scale_float: float
237
238
def forward(
239
self,
vllm/attention/backends/blocksparse_attn.py
+2
@@ -222,6 +222,7 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
222
slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
223
multi_modal_placeholder_index_maps=self.
224
multi_modal_placeholder_index_maps,
225
+ enable_kv_scales_calculation=self.enable_kv_scales_calculation,
226
seq_lens=self.seq_lens[:self.num_prefills],
227
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
228
max_query_len=self.max_query_len,
@@ -251,6 +252,7 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
252
num_decode_tokens=self.num_decode_tokens,
253
slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
254
multi_modal_placeholder_index_maps=None,
255
+ enable_kv_scales_calculation=False,
256
seq_lens=None,
257
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
258
max_query_len=None,
vllm/attention/backends/flash_attn.py
+4
-1
@@ -230,6 +230,7 @@ class FlashAttentionMetadata(AttentionMetadata):
230
slot_mapping=slot_mapping,
231
multi_modal_placeholder_index_maps=self.
232
multi_modal_placeholder_index_maps,
233
+ enable_kv_scales_calculation=self.enable_kv_scales_calculation,
234
seq_lens=seq_lens,
235
seq_lens_tensor=seq_lens_tensor,
236
max_query_len=self.max_query_len,
@@ -274,6 +275,7 @@ class FlashAttentionMetadata(AttentionMetadata):
275
num_decode_tokens=self.num_decode_tokens,
276
slot_mapping=slot_mapping,
277
multi_modal_placeholder_index_maps=None,
278
+ enable_kv_scales_calculation=True,
279
seq_lens=None,
280
seq_lens_tensor=seq_lens_tensor,
281
max_decode_query_len=self.max_decode_query_len,
@@ -557,6 +559,7 @@ class FlashAttentionMetadataBuilder(
559
num_decode_tokens=num_decode_tokens,
560
seq_lens=seq_lens,
561
multi_modal_placeholder_index_maps=placeholder_index_maps,
562
+ enable_kv_scales_calculation=True,
563
seq_lens_tensor=seq_lens_tensor,
564
max_query_len=max_query_len,
565
max_decode_query_len=max_decode_query_len,
@@ -675,7 +678,7 @@ class FlashAttentionImpl(AttentionImpl):
678
NOTE: It in-place updates the output tensor.
679
"""
680
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
678
- assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
681
+ assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0, (
682
"key/v_scale is not supported in FlashAttention.")
683
684
assert output is not None, "Output tensor must be provided."
vllm/attention/backends/flashinfer.py
+6
-4
@@ -219,6 +219,7 @@ class FlashInferState(AttentionState):
219
num_prefills=0,
220
slot_mapping=self._graph_slot_mapping[:batch_size],
221
multi_modal_placeholder_index_maps=None,
222
+ enable_kv_scales_calculation=False,
223
num_prefill_tokens=0,
224
num_decode_tokens=batch_size,
225
max_prefill_seq_len=0,
@@ -733,6 +734,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
734
num_prefills=self.num_prefills,
735
slot_mapping=slot_mapping_tensor,
736
multi_modal_placeholder_index_maps=placeholder_index_maps,
737
+ enable_kv_scales_calculation=False,
738
num_prefill_tokens=self.num_prefill_tokens,
739
num_decode_tokens=num_decode_tokens,
740
max_prefill_seq_len=max_prefill_seq_len,
@@ -888,8 +890,8 @@ class FlashInferImpl(AttentionImpl):
890
kv_cache,
891
logits_soft_cap=logits_soft_cap,
892
causal=True,
891
- k_scale=layer._k_scale,
892
- v_scale=layer._v_scale,
893
+ k_scale=layer._k_scale_float,
894
+ v_scale=layer._v_scale_float,
895
window_left=window_left)
896
if decode_meta := attn_metadata.decode_metadata:
897
assert decode_meta is not None
@@ -899,8 +901,8 @@ class FlashInferImpl(AttentionImpl):
901
kv_cache,
902
sm_scale=softmax_scale,
903
logits_soft_cap=logits_soft_cap,
902
- k_scale=layer._k_scale,
903
- v_scale=layer._v_scale,
904
+ k_scale=layer._k_scale_float,
905
+ v_scale=layer._v_scale_float,
906
window_left=window_left)
907
908
if prefill_output is None and decode_output is not None:
vllm/attention/backends/ipex_attn.py
+1
-1
@@ -193,7 +193,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
193
Returns:
194
shape = [num_tokens, num_heads * head_size]
195
"""
196
- assert layer._k_scale == 1.0 and layer._v_scale == 1.0
196
+ assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0
197
num_tokens, hidden_size = query.shape
198
# Reshape the query, key, and value tensors.
199
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/pallas.py
+1
-1
@@ -173,7 +173,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
173
Returns:
174
shape = [batch_size, seq_len, num_heads * head_size]
175
"""
176
- assert layer._k_scale == 1.0 and layer._v_scale == 1.0
176
+ assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0
177
batch_size, seq_len, hidden_size = query.shape
178
query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
179
key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/placeholder_attn.py
+3
@@ -140,6 +140,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
140
slot_mapping=slot_mapping,
141
multi_modal_placeholder_index_maps=self.
142
multi_modal_placeholder_index_maps,
143
+ enable_kv_scales_calculation=self.enable_kv_scales_calculation,
144
seq_lens=self.seq_lens[:self.num_prefills],
145
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
146
max_decode_query_len=0,
@@ -173,6 +174,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
174
num_decode_tokens=self.num_decode_tokens,
175
slot_mapping=slot_mapping,
176
multi_modal_placeholder_index_maps=None,
177
+ enable_kv_scales_calculation=True,
178
seq_lens=None,
179
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
180
max_decode_query_len=self.max_decode_query_len,
@@ -380,6 +382,7 @@ class PlaceholderAttentionMetadataBuilder(
382
num_prefills=self.num_prefills,
383
slot_mapping=slot_mapping,
384
multi_modal_placeholder_index_maps=placeholder_index_maps,
385
+ enable_kv_scales_calculation=True,
386
num_prefill_tokens=self.num_prefill_tokens,
387
num_decode_tokens=num_decode_tokens,
388
seq_lens=seq_lens,
vllm/attention/backends/rocm_flash_attn.py
+2
@@ -153,6 +153,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
153
slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
154
multi_modal_placeholder_index_maps=self.
155
multi_modal_placeholder_index_maps,
156
+ enable_kv_scales_calculation=self.enable_kv_scales_calculation,
157
seq_lens=self.seq_lens[:self.num_prefills],
158
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
159
max_query_len=self.max_query_len,
@@ -182,6 +183,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
183
num_decode_tokens=self.num_decode_tokens,
184
slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
185
multi_modal_placeholder_index_maps=None,
186
+ enable_kv_scales_calculation=True,
187
seq_lens=None,
188
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
189
max_query_len=None,
vllm/attention/backends/torch_sdpa.py
+1
-1
@@ -379,6 +379,7 @@ class TorchSDPAMetadataBuilder(AttentionMetadataBuilder[TorchSDPAMetadata]):
379
prefill_block_tables=prefill_block_tables,
380
slot_mapping=slot_mapping,
381
multi_modal_placeholder_index_maps=placeholder_index_maps,
382
+ enable_kv_scales_calculation=False,
383
)
384
385
return attn_metadata
@@ -454,7 +455,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
455
Returns:
456
shape = [num_tokens, num_heads * head_size]
457
"""
457
- assert layer._k_scale == 1.0 and layer._v_scale == 1.0
458
attn_type = self.attn_type
459
if (attn_type == AttentionType.ENCODER
460
and (not attn_metadata.is_all_encoder_attn_metadata_set)):
vllm/attention/backends/utils.py
+2
@@ -265,6 +265,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
265
num_prefills=self.num_prefills,
266
slot_mapping=slot_mapping_tensor,
267
multi_modal_placeholder_index_maps=placeholder_index_maps,
268
+ enable_kv_scales_calculation=True,
269
num_prefill_tokens=self.num_prefill_tokens,
270
num_decode_tokens=num_decode_tokens,
271
seq_lens=seq_lens,
@@ -317,6 +318,7 @@ class CommonAttentionState(AttentionState):
318
num_decode_tokens=batch_size,
319
slot_mapping=self._graph_slot_mapping[:batch_size],
320
multi_modal_placeholder_index_maps=None,
321
+ enable_kv_scales_calculation=True,
322
seq_lens=None,
323
seq_lens_tensor=self._graph_seq_lens[:batch_size],
324
max_query_len=1,
vllm/attention/backends/xformers.py
+2
@@ -218,6 +218,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
218
slot_mapping=slot_mapping,
219
multi_modal_placeholder_index_maps=self.
220
multi_modal_placeholder_index_maps,
221
+ enable_kv_scales_calculation=self.enable_kv_scales_calculation,
222
seq_lens=seq_lens,
223
seq_lens_tensor=seq_lens_tensor,
224
max_query_len=self.max_query_len,
@@ -262,6 +263,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
263
num_decode_tokens=self.num_decode_tokens,
264
slot_mapping=slot_mapping,
265
multi_modal_placeholder_index_maps=None,
266
+ enable_kv_scales_calculation=True,
267
seq_lens_tensor=seq_lens_tensor,
268
max_prefill_seq_len=0,
269
max_decode_seq_len=self.max_decode_seq_len,
vllm/attention/layer.py
+26
-2
@@ -5,6 +5,7 @@ import torch
5
import torch.nn as nn
6
import torch.nn.functional as F
7
8
+import vllm.envs as envs
9
from vllm.attention import AttentionMetadata, AttentionType
10
from vllm.attention.selector import backend_name_to_enum, get_attn_backend
11
from vllm.config import CacheConfig, get_current_vllm_config
@@ -57,10 +58,12 @@ class Attention(nn.Module):
58
kv_cache_dtype = cache_config.cache_dtype
59
block_size = cache_config.block_size
60
is_attention_free = cache_config.is_attention_free
61
+ calculate_kv_scales = cache_config.calculate_kv_scales
62
else:
63
kv_cache_dtype = "auto"
64
block_size = 16
65
is_attention_free = False
66
+ calculate_kv_scales = False
67
if num_kv_heads is None:
68
num_kv_heads = num_heads
69
@@ -70,8 +73,15 @@ class Attention(nn.Module):
73
# expect the pre-quantized k/v_scale to be loaded along
74
# with the model weights.
75
self.kv_cache_dtype = kv_cache_dtype
73
- self._k_scale = 1.0
74
- self._v_scale = 1.0
76
+ self.calculate_kv_scales = calculate_kv_scales
77
+ self._k_scale = torch.tensor(1.0, dtype=torch.float32)
78
+ self._v_scale = torch.tensor(1.0, dtype=torch.float32)
79
+
80
+ # We also keep the float32 versions of k/v_scale for attention
81
+ # backends that don't support tensors (Flashinfer)
82
+ self._k_scale_float = 1.0
83
+ self._v_scale_float = 1.0
84
+
85
quant_method = quant_config.get_quant_method(
86
self, prefix=prefix) if quant_config else None
87
if quant_method is not None:
@@ -127,6 +137,9 @@ class Attention(nn.Module):
137
).parallel_config.pipeline_parallel_size)
138
]
139
140
+ self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
141
+ self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
142
+
143
def forward(
144
self,
145
query: torch.Tensor,
@@ -135,6 +148,9 @@ class Attention(nn.Module):
148
kv_cache: torch.Tensor,
149
attn_metadata: AttentionMetadata,
150
) -> torch.Tensor:
151
+ if self.calculate_kv_scales and \
152
+ attn_metadata.enable_kv_scales_calculation:
153
+ self.calc_kv_scales(key, value)
154
if self.use_output:
155
output = torch.empty_like(query)
156
hidden_size = query.size(-1)
@@ -161,6 +177,14 @@ class Attention(nn.Module):
177
return torch.ops.vllm.unified_attention(
178
query, key, value, self.layer_name)
179
180
+ def calc_kv_scales(self, key, value):
181
+ self._k_scale.copy_(torch.abs(key).max() / self.k_range)
182
+ self._v_scale.copy_(torch.abs(value).max() / self.v_range)
183
+ self._k_scale_float = self._k_scale.item()
184
+ self._v_scale_float = self._v_scale.item()
185
+ # We only calculate the scales once
186
+ self.calculate_kv_scales = False
187
+
188
def extra_repr(self) -> str:
189
s = f"head_size={self.impl.head_size}" # type: ignore
190
s += f", num_heads={self.impl.num_heads}" # type: ignore
vllm/attention/ops/ipex_attn.py
+8
-8
@@ -52,8 +52,8 @@ class _PagedAttention:
52
value_cache: torch.Tensor,
53
slot_mapping: torch.Tensor,
54
kv_cache_dtype: str,
55
- k_scale: float,
56
- v_scale: float,
55
+ k_scale: torch.Tensor,
56
+ v_scale: torch.Tensor,
57
*args,
58
) -> None:
59
ops.reshape_and_cache(
@@ -80,8 +80,8 @@ class _PagedAttention:
80
num_kv_heads: int,
81
scale: float,
82
alibi_slopes: Optional[torch.Tensor],
83
- k_scale: float,
84
- v_scale: float,
83
+ k_scale: torch.Tensor,
84
+ v_scale: torch.Tensor,
85
*args,
86
) -> None:
87
tp_rank: int = 0
@@ -149,8 +149,8 @@ class _IPEXPagedAttention(_PagedAttention):
149
value_cache: torch.Tensor,
150
slot_mapping: torch.Tensor,
151
kv_cache_dtype: str,
152
- k_scale: float,
153
- v_scale: float,
152
+ k_scale: torch.Tensor,
153
+ v_scale: torch.Tensor,
154
*args,
155
) -> None:
156
ipex_modules.PagedAttention.reshape_and_cache(
@@ -170,8 +170,8 @@ class _IPEXPagedAttention(_PagedAttention):
170
num_kv_heads: int,
171
scale: float,
172
alibi_slopes: Optional[torch.Tensor],
173
- k_scale: float,
174
- v_scale: float,
173
+ k_scale: torch.Tensor,
174
+ v_scale: torch.Tensor,
175
*args,
176
) -> None:
177
block_size = value_cache.shape[2]
vllm/attention/ops/paged_attn.py
+6
-6
@@ -69,8 +69,8 @@ class PagedAttention:
69
value_cache: torch.Tensor,
70
slot_mapping: torch.Tensor,
71
kv_cache_dtype: str,
72
- k_scale: float,
73
- v_scale: float,
72
+ k_scale: torch.Tensor,
73
+ v_scale: torch.Tensor,
74
) -> None:
75
ops.reshape_and_cache(
76
key,
@@ -95,8 +95,8 @@ class PagedAttention:
95
num_kv_heads: int,
96
scale: float,
97
alibi_slopes: Optional[torch.Tensor],
98
- k_scale: float,
99
- v_scale: float,
98
+ k_scale: torch.Tensor,
99
+ v_scale: torch.Tensor,
100
tp_rank: int = 0,
101
blocksparse_local_blocks: int = 0,
102
blocksparse_vert_stride: int = 0,
@@ -204,8 +204,8 @@ class PagedAttention:
204
max_query_len: int,
205
alibi_slopes: Optional[torch.Tensor],
206
sliding_window: Optional[int],
207
- k_scale: float,
208
- v_scale: float,
207
+ k_scale: torch.Tensor,
208
+ v_scale: torch.Tensor,
209
) -> torch.Tensor:
210
output = torch.empty_like(query)
211
context_attention_fwd(
vllm/attention/ops/prefix_prefill.py
+6
-6
@@ -133,7 +133,7 @@ if triton.__version__ >= "2.1.0":
133
other=0.0) # [D,N]
134
135
if k_load.dtype.is_fp8():
136
- k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
136
+ k = (k_load.to(tl.float32) * tl.load(k_scale)).to(q.dtype)
137
else:
138
k = k_load
139
@@ -181,7 +181,7 @@ if triton.__version__ >= "2.1.0":
181
((start_n + offs_n[:, None]) < cur_batch_ctx_len),
182
other=0.0) # [N,D]
183
if v_load.dtype.is_fp8():
184
- v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
184
+ v = (v_load.to(tl.float32) * tl.load(v_scale)).to(q.dtype)
185
else:
186
v = v_load
187
p = p.to(v.dtype)
@@ -564,7 +564,7 @@ if triton.__version__ >= "2.1.0":
564
other=0.0) # [D,N]
565
566
if k_load.dtype.is_fp8():
567
- k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
567
+ k = (k_load.to(tl.float32) * tl.load(k_scale)).to(q.dtype)
568
else:
569
k = k_load
570
@@ -604,7 +604,7 @@ if triton.__version__ >= "2.1.0":
604
((start_n + offs_n[:, None]) < cur_batch_ctx_len),
605
other=0.0)
606
if v_load.dtype.is_fp8():
607
- v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
607
+ v = (v_load.to(tl.float32) * tl.load(v_scale)).to(q.dtype)
608
else:
609
v = v_load
610
p = p.to(v.dtype)
@@ -713,8 +713,8 @@ if triton.__version__ >= "2.1.0":
713
b_seq_len,
714
b_ctx_len,
715
max_input_len,
716
- k_scale: float = 1.0,
717
- v_scale: float = 1.0,
716
+ k_scale: torch.Tensor,
717
+ v_scale: torch.Tensor,
718
alibi_slopes=None,
719
sliding_window=None):
720
vllm/config.py
+6
-10
@@ -120,11 +120,6 @@ class ModelConfig:
120
decoding draft models.
121
quantization: Quantization method that was used to quantize the model
122
weights. If None, we assume the model weights are not quantized.
123
- quantization_param_path: Path to JSON file containing scaling factors.
124
- Used to load KV cache scaling factors into the model when KV cache
125
- type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also
126
- be used to load activation and weight scaling factors when the
127
- model dtype is FP8_E4M3 on ROCm.
123
enforce_eager: Whether to enforce eager execution. If True, we will
124
disable CUDA graph and always execute the model in eager mode.
125
If False, we will use CUDA graph and eager execution in hybrid.
@@ -187,7 +182,6 @@ class ModelConfig:
182
factors.append(self.model)
183
factors.append(self.dtype)
184
factors.append(self.quantization)
190
- factors.append(self.quantization_param_path)
185
factors.append(self.revision)
186
factors.append(self.code_revision)
187
factors.append(self.trust_remote_code)
@@ -213,7 +207,6 @@ class ModelConfig:
207
max_model_len: Optional[int] = None,
208
spec_target_max_model_len: Optional[int] = None,
209
quantization: Optional[str] = None,
216
- quantization_param_path: Optional[str] = None,
210
enforce_eager: Optional[bool] = None,
211
max_seq_len_to_capture: Optional[int] = None,
212
max_logprobs: int = 20,
@@ -274,7 +267,6 @@ class ModelConfig:
267
else:
268
self.tokenizer_revision = tokenizer_revision
269
self.quantization = quantization
277
- self.quantization_param_path = quantization_param_path
270
self.enforce_eager = enforce_eager
271
self.max_seq_len_to_capture = max_seq_len_to_capture
272
self.max_logprobs = max_logprobs
@@ -1002,6 +994,7 @@ class CacheConfig:
994
sliding_window: Optional[int] = None,
995
enable_prefix_caching: bool = False,
996
cpu_offload_gb: float = 0,
997
+ calculate_kv_scales: Optional[bool] = None,
998
) -> None:
999
self.block_size = block_size
1000
self.gpu_memory_utilization = gpu_memory_utilization
@@ -1012,7 +1005,7 @@ class CacheConfig:
1005
self.sliding_window = sliding_window
1006
self.enable_prefix_caching = enable_prefix_caching
1007
self.cpu_offload_gb = cpu_offload_gb
1015
-
1008
+ self.calculate_kv_scales = calculate_kv_scales
1009
self._verify_args()
1010
self._verify_cache_dtype()
1011
self._verify_prefix_caching()
@@ -1021,6 +1014,10 @@ class CacheConfig:
1014
self.num_gpu_blocks: Optional[int] = None
1015
self.num_cpu_blocks: Optional[int] = None
1016
1017
+ # Set calculate_kv_scales to False if the value is unset.
1018
+ if self.calculate_kv_scales is None:
1019
+ self.calculate_kv_scales = False
1020
+
1021
def metrics_info(self):
1022
# convert cache_config to dict(key: str, value: str) for prometheus
1023
# metrics info
@@ -3297,7 +3294,6 @@ class VllmConfig:
3294
f"quantization={self.model_config.quantization}, "
3295
f"enforce_eager={self.model_config.enforce_eager}, "
3296
f"kv_cache_dtype={self.cache_config.cache_dtype}, "
3300
- f"quantization_param_path={self.model_config.quantization_param_path},"
3297
f" device_config={self.device_config.device}, "
3298
f"decoding_config={self.decoding_config!r}, "
3299
f"observability_config={self.observability_config!r}, "
vllm/engine/arg_utils.py
+12
-13
@@ -98,7 +98,6 @@ class EngineArgs:
98
config_format: ConfigFormat = ConfigFormat.AUTO
99
dtype: str = 'auto'
100
kv_cache_dtype: str = 'auto'
101
- quantization_param_path: Optional[str] = None
101
seed: int = 0
102
max_model_len: Optional[int] = None
103
worker_use_ray: bool = False
@@ -199,6 +198,8 @@ class EngineArgs:
198
generation_config: Optional[str] = None
199
enable_sleep_mode: bool = False
200
201
+ calculate_kv_scales: Optional[bool] = None
202
+
203
def __post_init__(self):
204
if not self.tokenizer:
205
self.tokenizer = self.model
@@ -350,17 +351,6 @@ class EngineArgs:
351
help='Data type for kv cache storage. If "auto", will use model '
352
'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
353
'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
353
- parser.add_argument(
354
- '--quantization-param-path',
355
- type=nullable_str,
356
- default=None,
357
- help='Path to the JSON file containing the KV cache '
358
- 'scaling factors. This should generally be supplied, when '
359
- 'KV cache dtype is FP8. Otherwise, KV cache scaling factors '
360
- 'default to 1.0, which may cause accuracy issues. '
361
- 'FP8_E5M2 (without scaling) is only supported on cuda version '
362
- 'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
363
- 'supported for common inference criteria.')
354
parser.add_argument('--max-model-len',
355
type=int,
356
default=EngineArgs.max_model_len,
@@ -962,6 +952,15 @@ class EngineArgs:
952
help="Enable sleep mode for the engine. "
953
"(only cuda platform is supported)")
954
955
+ parser.add_argument(
956
+ '--calculate-kv-scales',
957
+ action='store_true',
958
+ help='This enables dynamic calculation of '
959
+ 'k_scale and v_scale when kv-cache-dtype is fp8. '
960
+ 'If calculate-kv-scales is false, the scales will '
961
+ 'be loaded from the model checkpoint if available. '
962
+ 'Otherwise, the scales will default to 1.0.')
963
+
964
return parser
965
966
@classmethod
@@ -991,7 +990,6 @@ class EngineArgs:
990
tokenizer_revision=self.tokenizer_revision,
991
max_model_len=self.max_model_len,
992
quantization=self.quantization,
994
- quantization_param_path=self.quantization_param_path,
993
enforce_eager=self.enforce_eager,
994
max_seq_len_to_capture=self.max_seq_len_to_capture,
995
max_logprobs=self.max_logprobs,
@@ -1068,6 +1066,7 @@ class EngineArgs:
1066
sliding_window=model_config.get_sliding_window(),
1067
enable_prefix_caching=self.enable_prefix_caching,
1068
cpu_offload_gb=self.cpu_offload_gb,
1069
+ calculate_kv_scales=self.calculate_kv_scales,
1070
)
1071
parallel_config = ParallelConfig(
1072
pipeline_parallel_size=self.pipeline_parallel_size,
vllm/envs.py
+9
@@ -73,6 +73,8 @@ if TYPE_CHECKING:
73
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
74
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
75
VLLM_DISABLE_COMPILE_CACHE: bool = False
76
+ K_SCALE_CONSTANT: int = 200
77
+ V_SCALE_CONSTANT: int = 100
78
VLLM_SERVER_DEV_MODE: bool = False
79
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
80
@@ -474,6 +476,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
476
"VLLM_USE_V1":
477
lambda: bool(int(os.getenv("VLLM_USE_V1", "0"))),
478
479
+ # Divisor for dynamic key scale factor calculation for FP8 KV Cache
480
+ "K_SCALE_CONSTANT":
481
+ lambda: int(os.getenv("K_SCALE_CONSTANT", "200")),
482
+
483
+ # Divisor for dynamic value scale factor calculation for FP8 KV Cache
484
+ "V_SCALE_CONSTANT":
485
+ lambda: int(os.getenv("V_SCALE_CONSTANT", "100")),
486
# If set, enable multiprocessing in LLM for the V1 code path.
487
"VLLM_ENABLE_V1_MULTIPROCESSING":
488
lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))),
vllm/model_executor/layers/quantization/kv_cache.py
+15
-4
@@ -3,6 +3,7 @@ import torch
3
from vllm.logger import init_logger
4
from vllm.model_executor.layers.quantization.base_config import (
5
QuantizationConfig, QuantizeMethodBase)
6
+from vllm.platforms import current_platform
7
8
logger = init_logger(__name__)
9
@@ -40,11 +41,16 @@ class BaseKVCacheMethod(QuantizeMethodBase):
41
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
42
# If the kv-cache dtype is auto, we enforce the k/v_scale to be 1.0
43
# regardless whether the kv-scale is available in the checkpoint.
43
- if layer.kv_cache_dtype != "auto":
44
+ # No need to process kv scales after loading if we are going to
45
+ # calculate them on the fly.
46
+ if layer.kv_cache_dtype != "auto" and not layer.calculate_kv_scales:
47
if layer.k_scale > 0.0 and layer.v_scale > 0.0:
48
# We prefer to use separate k_scale and v_scale if present
49
k_scale = layer.k_scale.to("cpu").tolist()
50
v_scale = layer.v_scale.to("cpu").tolist()
51
+ if current_platform.is_rocm():
52
+ k_scale *= 2
53
+ v_scale *= 2
54
elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
55
# If no scales were loaded (both scales are invalid negative
56
# values), use the default value of 1.0
@@ -58,6 +64,9 @@ class BaseKVCacheMethod(QuantizeMethodBase):
64
scale_to_duplicate = max(layer.k_scale, layer.v_scale)
65
k_scale = scale_to_duplicate.to("cpu").tolist()
66
v_scale = scale_to_duplicate.to("cpu").tolist()
67
+ if current_platform.is_rocm():
68
+ k_scale *= 2
69
+ v_scale *= 2
70
71
if not isinstance(k_scale, float) or not isinstance(
72
v_scale, float):
@@ -65,9 +74,11 @@ class BaseKVCacheMethod(QuantizeMethodBase):
74
"for fp8 KV cache")
75
76
# These are used in the final Attention.forward()
68
- layer._k_scale = k_scale
69
- layer._v_scale = v_scale
70
- if (layer._k_scale == 1.0 and layer._v_scale == 1.0
77
+ layer._k_scale.copy_(k_scale)
78
+ layer._v_scale.copy_(v_scale)
79
+ layer._k_scale_float = k_scale
80
+ layer._v_scale_float = v_scale
81
+ if (k_scale == 1.0 and v_scale == 1.0
82
and "e5m2" not in layer.kv_cache_dtype):
83
logger.warning_once(
84
"Using KV cache scaling factor 1.0 for fp8_e4m3. This "
vllm/model_executor/model_loader/weight_utils.py
+1
-44
@@ -6,8 +6,7 @@ import json
6
import os
7
import tempfile
8
from collections import defaultdict
9
-from typing import (Any, Callable, Dict, Generator, Iterable, List, Optional,
10
- Tuple, Union)
9
+from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
10
11
import filelock
12
import gguf
@@ -23,7 +22,6 @@ from vllm.distributed import get_tensor_model_parallel_rank
22
from vllm.logger import init_logger
23
from vllm.model_executor.layers.quantization import (QuantizationConfig,
24
get_quantization_config)
26
-from vllm.model_executor.layers.quantization.schema import QuantParamSchema
25
from vllm.platforms import current_platform
26
from vllm.utils import PlaceholderModule
27
@@ -496,47 +494,6 @@ def gguf_quant_weights_iterator(
494
yield name, param
495
496
499
-def kv_cache_scales_loader(
500
- filename: str, tp_rank: int, tp_size: int, num_hidden_layers: int,
501
- model_type: Optional[str]) -> Iterable[Tuple[int, float]]:
502
- """
503
- A simple utility to read in KV cache scaling factors that have been
504
- previously serialized to disk. Used by the model to populate the appropriate
505
- KV cache scaling factors. The serialization should represent a dictionary
506
- whose keys are the TP ranks and values are another dictionary mapping layers
507
- to their KV cache scaling factors.
508
- Keep this function in sync with the output of
509
- examples/other/fp8/extract_scales.py
510
- """
511
- try:
512
- with open(filename) as f:
513
- context = {
514
- "model_type": model_type,
515
- "num_hidden_layers": num_hidden_layers,
516
- "tp_rank": tp_rank,
517
- "tp_size": tp_size,
518
- }
519
- schema_dct = json.load(f)
520
- schema = QuantParamSchema.model_validate(schema_dct,
521
- context=context)
522
- layer_scales_map = schema.kv_cache.scaling_factor[tp_rank]
523
- return layer_scales_map.items()
524
-
525
- except FileNotFoundError:
526
- logger.error("File or directory '%s' not found.", filename)
527
- except json.JSONDecodeError:
528
- logger.error("Error decoding JSON in file '%s'.", filename)
529
- except Exception:
530
- logger.exception("An error occurred while reading '%s'.", filename)
531
- # This section is reached if and only if any of the excepts are hit
532
- # Return an empty iterable (list) => no KV cache scales are loaded
533
- # which ultimately defaults to 1.0 scales
534
- logger.warning(
535
- "Defaulting to KV cache scaling factors = 1.0 for all "
536
- "layers in TP rank %d as an error occurred during loading.", tp_rank)
537
- return []
538
-
539
-
497
def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
498
"""convert PySafeSlice object from safetensors to torch.Tensor
499
vllm/model_executor/models/exaone.py
+2
-33
@@ -30,8 +30,7 @@ from torch import nn
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, VllmConfig
33
-from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
- get_tensor_model_parallel_world_size)
33
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -44,9 +43,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import (
47
- default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
46
+ default_weight_loader, maybe_remap_kv_scale_name)
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
-from vllm.platforms import current_platform
48
from vllm.sequence import IntermediateTensors
49
from vllm.transformers_utils.configs.exaone import ExaoneConfig
50
@@ -576,32 +574,3 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
574
weight_loader(param, loaded_weight)
575
loaded_params.add(name)
576
return loaded_params
579
-
580
- # If this function is called, it should always initialize KV cache scale
581
- # factors (or else raise an exception). Thus, handled exceptions should
582
- # make sure to leave KV cache scale factors in a known good (dummy) state
583
- def load_kv_cache_scales(self, quantization_param_path: str) -> None:
584
- tp_size = get_tensor_model_parallel_world_size()
585
- tp_rank = get_tensor_model_parallel_rank()
586
- for layer_idx, scaling_factor in kv_cache_scales_loader(
587
- quantization_param_path,
588
- tp_rank,
589
- tp_size,
590
- self.config.num_hidden_layers,
591
- self.config.__class__.model_type,
592
- ):
593
- if not isinstance(self.transformer.h[layer_idx], nn.Identity):
594
- layer_self_attn = self.transformer.h[layer_idx].attn
595
-
596
- if current_platform.is_rocm():
597
- # The scaling factor convention we are assuming is
598
- # quantized_value * scaling_factor ~= true_value
599
- # which is consistent with the practice of setting
600
- # scaling_factor = tensor_amax / FPtype_max
601
- scaling_factor *= 2
602
- if hasattr(layer_self_attn.attn, "_k_scale"):
603
- layer_self_attn.attn._k_scale = scaling_factor
604
- layer_self_attn.attn._v_scale = scaling_factor
605
- else:
606
- raise RuntimeError("Self attention has no KV cache scaling "
607
- "factor attribute!")
vllm/model_executor/models/granite.py
+2
-30
@@ -29,8 +29,7 @@ from transformers import GraniteConfig
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
from vllm.config import CacheConfig, VllmConfig
32
-from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
- get_tensor_model_parallel_world_size)
32
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -44,9 +43,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import (
47
- default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
46
+ default_weight_loader, maybe_remap_kv_scale_name)
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
-from vllm.platforms import current_platform
48
from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsLoRA, SupportsPP
@@ -518,29 +516,3 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
516
weight_loader(param, loaded_weight)
517
loaded_params.add(name)
518
return loaded_params
521
-
522
- # If this function is called, it should always initialize KV cache scale
523
- # factors (or else raise an exception). Thus, handled exceptions should
524
- # make sure to leave KV cache scale factors in a known good (dummy) state
525
- def load_kv_cache_scales(self, quantization_param_path: str) -> None:
526
- tp_size = get_tensor_model_parallel_world_size()
527
- tp_rank = get_tensor_model_parallel_rank()
528
- for layer_idx, scaling_factor in kv_cache_scales_loader(
529
- quantization_param_path, tp_rank, tp_size,
530
- self.config.num_hidden_layers,
531
- self.config.__class__.model_type):
532
- if not isinstance(self.model.layers[layer_idx], nn.Identity):
533
- layer_self_attn = self.model.layers[layer_idx].self_attn
534
-
535
- if current_platform.is_rocm():
536
- # The scaling factor convention we are assuming is
537
- # quantized_value * scaling_factor ~= true_value
538
- # which is consistent with the practice of setting
539
- # scaling_factor = tensor_amax / FPtype_max
540
- scaling_factor *= 2
541
- if hasattr(layer_self_attn.attn, "_k_scale"):
542
- layer_self_attn.attn._k_scale = scaling_factor
543
- layer_self_attn.attn._v_scale = scaling_factor
544
- else:
545
- raise RuntimeError("Self attention has no KV cache scaling "
546
- "factor attribute!")
vllm/model_executor/models/llama.py
+2
-33
@@ -29,8 +29,7 @@ from transformers import LlamaConfig
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.compilation.decorators import support_torch_compile
31
from vllm.config import CacheConfig, VllmConfig
32
-from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
- get_tensor_model_parallel_world_size)
32
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -43,9 +42,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import (
46
- default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
45
+ default_weight_loader, maybe_remap_kv_scale_name)
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
-from vllm.platforms import current_platform
47
from vllm.sequence import IntermediateTensors
48
49
from .interfaces import SupportsLoRA, SupportsPP
@@ -440,32 +438,6 @@ class LlamaModel(nn.Module):
438
loaded_params.add(name)
439
return loaded_params
440
443
- # If this function is called, it should always initialize KV cache scale
444
- # factors (or else raise an exception). Thus, handled exceptions should
445
- # make sure to leave KV cache scale factors in a known good (dummy) state
446
- def load_kv_cache_scales(self, quantization_param_path: str) -> None:
447
- tp_size = get_tensor_model_parallel_world_size()
448
- tp_rank = get_tensor_model_parallel_rank()
449
- for layer_idx, scaling_factor in kv_cache_scales_loader(
450
- quantization_param_path, tp_rank, tp_size,
451
- self.config.num_hidden_layers,
452
- self.config.__class__.model_type):
453
- if not isinstance(self.layers[layer_idx], nn.Identity):
454
- layer_self_attn = self.layers[layer_idx].self_attn
455
-
456
- if current_platform.is_rocm():
457
- # The scaling factor convention we are assuming is
458
- # quantized_value * scaling_factor ~= true_value
459
- # which is consistent with the practice of setting
460
- # scaling_factor = tensor_amax / FPtype_max
461
- scaling_factor *= 2
462
- if hasattr(layer_self_attn.attn, "_k_scale"):
463
- layer_self_attn.attn._k_scale = scaling_factor
464
- layer_self_attn.attn._v_scale = scaling_factor
465
- else:
466
- raise RuntimeError("Self attention has no KV cache scaling "
467
- "factor attribute!")
468
-
441
442
class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
443
packed_modules_mapping = {
@@ -593,9 +565,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
565
self.maybe_remap_mistral(name, loaded_weight)
566
for name, loaded_weight in weights)
567
596
- def load_kv_cache_scales(self, quantization_param_path: str) -> None:
597
- self.model.load_kv_cache_scales(quantization_param_path)
598
-
568
# This function is used to remap the mistral format as
569
# used by Mistral and Llama <=2
570
def maybe_remap_mistral(
vllm/model_executor/models/mllama.py
+4
-3
@@ -831,6 +831,7 @@ class MllamaTextCrossAttention(nn.Module):
831
) -> torch.Tensor:
832
# Skip writing kv-cache for the initial profiling run.
833
if len(kv_cache.shape) > 1:
834
+ i = torch.ones(1, dtype=torch.float32)
835
if self.attn.backend in (_Backend.FLASH_ATTN,
836
_Backend.FLASH_ATTN_VLLM_V1):
837
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
@@ -843,8 +844,8 @@ class MllamaTextCrossAttention(nn.Module):
844
attn_metadata.
845
cross_slot_mapping, # type: ignore[union-attr]
846
"auto",
846
- 1.0,
847
- 1.0,
847
+ i,
848
+ i,
849
)
850
elif self.attn.backend in (_Backend.XFORMERS, _Backend.TORCH_SDPA):
851
key_cache, value_cache = PagedAttention.split_kv_cache(
@@ -853,7 +854,7 @@ class MllamaTextCrossAttention(nn.Module):
854
cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
855
PagedAttention.write_to_paged_cache(
856
cached_k, cached_v, key_cache, value_cache,
856
- attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
857
+ attn_metadata.cross_slot_mapping, "auto", i, i)
858
else:
859
raise ValueError(
860
f"Unsupported Attention backend {self.attn.backend} "
vllm/model_executor/models/solar.py
+2
-33
@@ -30,8 +30,7 @@ from transformers import PretrainedConfig
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, VllmConfig
33
-from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
- get_tensor_model_parallel_world_size)
33
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -44,9 +43,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import (
47
- default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
46
+ default_weight_loader, maybe_remap_kv_scale_name)
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
-from vllm.platforms import current_platform
48
from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsLoRA, SupportsPP
@@ -535,32 +533,3 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
533
weight_loader(param, loaded_weight)
534
loaded_params.add(name)
535
return loaded_params
538
-
539
- # If this function is called, it should always initialize KV cache scale
540
- # factors (or else raise an exception). Thus, handled exceptions should
541
- # make sure to leave KV cache scale factors in a known good (dummy) state
542
- def load_kv_cache_scales(self, quantization_param_path: str) -> None:
543
- tp_size = get_tensor_model_parallel_world_size()
544
- tp_rank = get_tensor_model_parallel_rank()
545
- for layer_idx, scaling_factor in kv_cache_scales_loader(
546
- quantization_param_path,
547
- tp_rank,
548
- tp_size,
549
- self.config.num_hidden_layers,
550
- self.config.__class__.model_type,
551
- ):
552
- if not isinstance(self.model.layers[layer_idx], nn.Identity):
553
- layer_self_attn = self.model.layers[layer_idx].self_attn
554
-
555
- if current_platform.is_rocm():
556
- # The scaling factor convention we are assuming is
557
- # quantized_value * scaling_factor ~= true_value
558
- # which is consistent with the practice of setting
559
- # scaling_factor = tensor_amax / FPtype_max
560
- scaling_factor *= 2
561
- if hasattr(layer_self_attn.attn, "_k_scale"):
562
- layer_self_attn.attn._k_scale = scaling_factor
563
- layer_self_attn.attn._v_scale = scaling_factor
564
- else:
565
- raise RuntimeError("Self attention has no KV cache scaling "
566
- "factor attribute!")
vllm/v1/attention/backends/flash_attn.py
-4
@@ -166,10 +166,6 @@ class FlashAttentionImpl(AttentionImpl):
166
Returns:
167
shape = [num_tokens, num_heads * head_size]
168
"""
169
- # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
170
- assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
171
- "key/v_scale is not supported in FlashAttention.")
172
-
169
assert output is not None, "Output tensor must be provided."
170
171
if attn_metadata is None:
vllm/worker/hpu_model_runner.py
+5
-2
@@ -903,7 +903,8 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
903
num_decode_tokens=0,
904
slot_mapping=slot_mapping,
905
multi_modal_placeholder_index_maps=
906
- None # FIXME(kzawora): mutli-modality will not work here
906
+ None, # FIXME(kzawora): mutli-modality will not work here
907
+ enable_kv_scales_calculation=False,
908
)
909
multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
910
@@ -1057,7 +1058,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
1058
num_prefill_tokens=0,
1059
num_decode_tokens=num_decode_tokens,
1060
slot_mapping=slot_mapping,
1060
- multi_modal_placeholder_index_maps=None)
1061
+ multi_modal_placeholder_index_maps=None,
1062
+ enable_kv_scales_calculation=False,
1063
+ )
1064
return PrepareDecodeMetadata(input_tokens=input_tokens,
1065
input_positions=input_positions,
1066
attn_metadata=attn_metadata,
vllm/worker/model_runner.py
+6
-31
@@ -3,7 +3,6 @@ import gc
3
import inspect
4
import itertools
5
import time
6
-import warnings
6
import weakref
7
from contextlib import contextmanager
8
from dataclasses import dataclass
@@ -41,7 +40,6 @@ from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
40
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
41
MultiModalKwargs, MultiModalPlaceholderMap,
42
MultiModalRegistry)
44
-from vllm.platforms import current_platform
43
from vllm.prompt_adapter.layers import PromptAdapterMapping
44
from vllm.prompt_adapter.request import PromptAdapterRequest
45
from vllm.prompt_adapter.worker_manager import (
@@ -1151,34 +1149,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1149
self.prompt_adapter_manager.create_prompt_adapter_manager(
1150
self.model))
1151
1154
- if self.kv_cache_dtype == "fp8" and (current_platform.is_rocm()
1155
- or current_platform.is_cuda()):
1156
- # Currently only ROCm accepts kv-cache scaling factors
1157
- # via quantization_param_path and this will be deprecated
1158
- # in the future.
1159
- if self.model_config.quantization_param_path is not None:
1160
- if callable(getattr(self.model, "load_kv_cache_scales", None)):
1161
- warnings.warn(
1162
- "Loading kv cache scaling factor from JSON is "
1163
- "deprecated and will be removed. Please include "
1164
- "kv cache scaling factors in the model checkpoint.",
1165
- FutureWarning,
1166
- stacklevel=2)
1167
- self.model.load_kv_cache_scales(
1168
- self.model_config.quantization_param_path)
1169
- logger.info("Loaded KV cache scaling factors from %s",
1170
- self.model_config.quantization_param_path)
1171
- else:
1172
- raise RuntimeError(
1173
- "Using FP8 KV cache and scaling factors provided but "
1174
- "model %s does not support loading scaling factors.",
1175
- self.model.__class__)
1176
- else:
1177
- logger.warning(
1178
- "Using FP8 KV cache but no scaling factors "
1179
- "provided. Defaulting to scaling factors of 1.0. "
1180
- "This may lead to less accurate results!")
1181
-
1152
if self.vllm_config.compilation_config.level ==\
1153
CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1154
backend = self.vllm_config.compilation_config.init_backend(
@@ -1366,6 +1336,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1336
dtype=self.model_config.dtype,
1337
device=self.device)
1338
1339
+ # Disable KV Scale Calculation for dummy data during profile run
1340
+ if model_input.attn_metadata is not None:
1341
+ model_input.attn_metadata.enable_kv_scales_calculation = False
1342
+
1343
self.execute_model(model_input, kv_caches, intermediate_tensors)
1344
torch.cuda.synchronize()
1345
return
@@ -1510,7 +1484,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1484
batch_size,
1485
is_encoder_decoder_model=self.model_config.
1486
is_encoder_decoder))
1513
-
1487
+ # Disable KV Scale Calculation for graph capture
1488
+ attn_metadata.enable_kv_scales_calculation = False
1489
if self.lora_config:
1490
lora_mapping = LoRAMapping(
1491
**dict(index_mapping=[0] * batch_size,
vllm/worker/openvino_model_runner.py
+1
@@ -282,6 +282,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
282
block_indices_begins=block_indices_begins_tensor,
283
max_context_len=max_context_len_tensor,
284
multi_modal_placeholder_index_maps=placeholder_index_maps,
285
+ enable_kv_scales_calculation=False,
286
)
287
288
multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
vllm/worker/tpu_model_runner.py
+5
@@ -190,6 +190,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
190
num_decode_tokens=0,
191
slot_mapping=slot_mapping,
192
multi_modal_placeholder_index_maps=None,
193
+ enable_kv_scales_calculation=False,
194
block_tables=None,
195
context_lens=None,
196
effective_query_lens=None,
@@ -208,6 +209,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
209
num_decode_tokens=0,
210
slot_mapping=slot_mapping,
211
multi_modal_placeholder_index_maps=None,
212
+ enable_kv_scales_calculation=False,
213
block_tables=block_tables,
214
context_lens=context_lens,
215
effective_query_lens=effective_query_lens,
@@ -239,6 +241,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
241
num_decode_tokens=batch_size * seq_len,
242
slot_mapping=slot_mapping,
243
multi_modal_placeholder_index_maps=None,
244
+ enable_kv_scales_calculation=False,
245
block_tables=block_tables,
246
context_lens=context_lens,
247
)
@@ -425,6 +428,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
428
num_decode_tokens=0,
429
slot_mapping=slot_mapping,
430
multi_modal_placeholder_index_maps=None,
431
+ enable_kv_scales_calculation=False,
432
block_tables=block_tables,
433
context_lens=context_lens,
434
effective_query_lens=prompt_lens,
@@ -496,6 +500,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
500
num_decode_tokens=batch_size,
501
slot_mapping=slot_mapping,
502
multi_modal_placeholder_index_maps=None,
503
+ enable_kv_scales_calculation=False,
504
block_tables=block_tables,
505
context_lens=context_lens,
506
)
vllm/worker/xpu_model_runner.py
+2
@@ -261,6 +261,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
261
is_prompt=True,
262
slot_mapping=slot_mapping,
263
multi_modal_placeholder_index_maps=placeholder_index_maps,
264
+ enable_kv_scales_calculation=False,
265
seq_lens=seq_lens,
266
seqlen_q=seqlen_q,
267
max_seqlen=max_seqlen,
@@ -345,6 +346,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
346
is_prompt=False,
347
slot_mapping=slot_mapping,
348
multi_modal_placeholder_index_maps=None,
349
+ enable_kv_scales_calculation=False,
350
seq_lens=seq_lens,
351
seqlen_q=torch.tensor([]),
352
max_seqlen=0,