@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e97f802b2d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[FP8][Kernel] Dynamic kv cache scaling factors computation (#11906)

Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com> Co-authored-by: Micah Williamson <micah.williamson@amd.com>

Gregory Shtrasberg committed Jan 23, 2025 at 13:04 UTC e97f802b2d74861af77997691a7d1c36498f6dca
60 files changed +276 -1365
benchmarks/kernels/benchmark_paged_attention.py
+3 -1
@@ -98,7 +98,9 @@ def main(
98 start_time = time.perf_counter()
99
100 # Using default kv_scale
101 - k_scale = v_scale = 1.0
101 + k_scale = v_scale = torch.tensor(1.0,
102 + dtype=torch.float32,
103 + device=device)
104
105 for _ in range(num_iters):
106 if version == "v1":
csrc/attention/attention_kernels.cuh
+5 -5
@@ -105,7 +105,7 @@ __device__ void paged_attention_kernel(
105 const int max_num_blocks_per_seq,
106 const float* __restrict__ alibi_slopes, // [num_heads]
107 const int q_stride, const int kv_block_stride, const int kv_head_stride,
108 - const float k_scale, const float v_scale, const int tp_rank,
108 + const float* k_scale, const float* v_scale, const int tp_rank,
109 const int blocksparse_local_blocks, const int blocksparse_vert_stride,
110 const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
111 const int seq_idx = blockIdx.y;
@@ -285,7 +285,7 @@ __device__ void paged_attention_kernel(
285 Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(
286 k_ptr + offset1 * BLOCK_SIZE * x + offset2);
287 k_vecs[j] = fp8::scaled_convert<K_vec, Quant_vec, KV_DTYPE>(
288 - k_vec_quant, k_scale);
288 + k_vec_quant, *k_scale);
289 }
290 }
291
@@ -415,7 +415,7 @@ __device__ void paged_attention_kernel(
415 *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
416 // Vector conversion from V_quant_vec to V_vec.
417 v_vec = fp8::scaled_convert<V_vec, V_quant_vec, KV_DTYPE>(v_quant_vec,
418 - v_scale);
418 + *v_scale);
419 }
420 if (block_idx == num_seq_blocks - 1) {
421 // NOTE(woosuk): When v_vec contains the tokens that are out of the
@@ -513,7 +513,7 @@ __global__ void paged_attention_v1_kernel(
513 const int max_num_blocks_per_seq,
514 const float* __restrict__ alibi_slopes, // [num_heads]
515 const int q_stride, const int kv_block_stride, const int kv_head_stride,
516 - const float k_scale, const float v_scale, const int tp_rank,
516 + const float* k_scale, const float* v_scale, const int tp_rank,
517 const int blocksparse_local_blocks, const int blocksparse_vert_stride,
518 const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
519 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
@@ -549,7 +549,7 @@ __global__ void paged_attention_v2_kernel(
549 const int max_num_blocks_per_seq,
550 const float* __restrict__ alibi_slopes, // [num_heads]
551 const int q_stride, const int kv_block_stride, const int kv_head_stride,
552 - const float k_scale, const float v_scale, const int tp_rank,
552 + const float* k_scale, const float* v_scale, const int tp_rank,
553 const int blocksparse_local_blocks, const int blocksparse_vert_stride,
554 const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
555 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
csrc/attention/paged_attention_v1.cu
+10 -7
@@ -41,7 +41,7 @@
41 out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
42 scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
43 alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
44 - k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
44 + k_scale_ptr, v_scale_ptr, tp_rank, blocksparse_local_blocks, \
45 blocksparse_vert_stride, blocksparse_block_size, \
46 blocksparse_head_sliding_step);
47
@@ -53,10 +53,10 @@ void paged_attention_v1_launcher(
53 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
54 torch::Tensor& value_cache, int num_kv_heads, float scale,
55 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
56 - const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
57 - float v_scale, const int tp_rank, const int blocksparse_local_blocks,
58 - const int blocksparse_vert_stride, const int blocksparse_block_size,
59 - const int blocksparse_head_sliding_step) {
56 + const std::optional<torch::Tensor>& alibi_slopes, torch::Tensor& k_scale,
57 + torch::Tensor& v_scale, const int tp_rank,
58 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
59 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
60 int num_seqs = query.size(0);
61 int num_heads = query.size(1);
62 int head_size = query.size(2);
@@ -80,6 +80,8 @@ void paged_attention_v1_launcher(
80 CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
81 int* block_tables_ptr = block_tables.data_ptr<int>();
82 int* seq_lens_ptr = seq_lens.data_ptr<int>();
83 + const float* k_scale_ptr = reinterpret_cast<const float*>(k_scale.data_ptr());
84 + const float* v_scale_ptr = reinterpret_cast<const float*>(v_scale.data_ptr());
85
86 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
87 int padded_max_seq_len =
@@ -177,8 +179,9 @@ void paged_attention_v1(
179 torch::Tensor& seq_lens, // [num_seqs]
180 int64_t block_size, int64_t max_seq_len,
181 const std::optional<torch::Tensor>& alibi_slopes,
180 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
181 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
182 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
183 + torch::Tensor& v_scale, const int64_t tp_rank,
184 + const int64_t blocksparse_local_blocks,
185 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
186 const int64_t blocksparse_head_sliding_step) {
187 const bool is_block_sparse = (blocksparse_vert_stride > 1);
csrc/attention/paged_attention_v2.cu
+10 -7
@@ -37,7 +37,7 @@
37 exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
38 value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
39 seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
40 - kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
40 + kv_block_stride, kv_head_stride, k_scale_ptr, v_scale_ptr, tp_rank, \
41 blocksparse_local_blocks, blocksparse_vert_stride, \
42 blocksparse_block_size, blocksparse_head_sliding_step); \
43 vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
@@ -54,10 +54,10 @@ void paged_attention_v2_launcher(
54 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
55 torch::Tensor& value_cache, int num_kv_heads, float scale,
56 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
57 - const std::optional<torch::Tensor>& alibi_slopes, float k_scale,
58 - float v_scale, const int tp_rank, const int blocksparse_local_blocks,
59 - const int blocksparse_vert_stride, const int blocksparse_block_size,
60 - const int blocksparse_head_sliding_step) {
57 + const std::optional<torch::Tensor>& alibi_slopes, torch::Tensor& k_scale,
58 + torch::Tensor& v_scale, const int tp_rank,
59 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
60 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
61 int num_seqs = query.size(0);
62 int num_heads = query.size(1);
63 int head_size = query.size(2);
@@ -84,6 +84,8 @@ void paged_attention_v2_launcher(
84 CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
85 int* block_tables_ptr = block_tables.data_ptr<int>();
86 int* seq_lens_ptr = seq_lens.data_ptr<int>();
87 + const float* k_scale_ptr = reinterpret_cast<const float*>(k_scale.data_ptr());
88 + const float* v_scale_ptr = reinterpret_cast<const float*>(v_scale.data_ptr());
89
90 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
91 int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
@@ -188,8 +190,9 @@ void paged_attention_v2(
190 torch::Tensor& seq_lens, // [num_seqs]
191 int64_t block_size, int64_t max_seq_len,
192 const std::optional<torch::Tensor>& alibi_slopes,
191 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
192 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
193 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
194 + torch::Tensor& v_scale, const int64_t tp_rank,
195 + const int64_t blocksparse_local_blocks,
196 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
197 const int64_t blocksparse_head_sliding_step) {
198 const bool is_block_sparse = (blocksparse_vert_stride > 1);
csrc/cache.h
+3 -3
@@ -18,15 +18,15 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
18 void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
19 torch::Tensor& key_cache, torch::Tensor& value_cache,
20 torch::Tensor& slot_mapping,
21 - const std::string& kv_cache_dtype, const double k_scale,
22 - const double v_scale);
21 + const std::string& kv_cache_dtype,
22 + torch::Tensor& k_scale, torch::Tensor& v_scale);
23
24 void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
25 torch::Tensor& key_cache,
26 torch::Tensor& value_cache,
27 torch::Tensor& slot_mapping,
28 const std::string& kv_cache_dtype,
29 - const double k_scale, const double v_scale);
29 + torch::Tensor& k_scale, torch::Tensor& v_scale);
30
31 // Just for unittest
32 void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
csrc/cache_kernels.cu
+17 -13
@@ -159,8 +159,8 @@ __global__ void reshape_and_cache_kernel(
159 // block_size]
160 const int64_t* __restrict__ slot_mapping, // [num_tokens]
161 const int key_stride, const int value_stride, const int num_heads,
162 - const int head_size, const int block_size, const int x, const float k_scale,
163 - const float v_scale) {
162 + const int head_size, const int block_size, const int x,
163 + const float* k_scale, const float* v_scale) {
164 const int64_t token_idx = blockIdx.x;
165 const int64_t slot_idx = slot_mapping[token_idx];
166 if (slot_idx < 0) {
@@ -196,9 +196,9 @@ __global__ void reshape_and_cache_kernel(
196 value_cache[tgt_value_idx] = tgt_value;
197 } else {
198 key_cache[tgt_key_idx] =
199 - fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, k_scale);
199 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, *k_scale);
200 value_cache[tgt_value_idx] =
201 - fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, v_scale);
201 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, *v_scale);
202 }
203 }
204 }
@@ -214,7 +214,7 @@ __global__ void reshape_and_cache_flash_kernel(
214 const int64_t* __restrict__ slot_mapping, // [num_tokens]
215 const int block_stride, const int key_stride, const int value_stride,
216 const int num_heads, const int head_size, const int block_size,
217 - const float k_scale, const float v_scale) {
217 + const float* k_scale, const float* v_scale) {
218 const int64_t token_idx = blockIdx.x;
219 const int64_t slot_idx = slot_mapping[token_idx];
220 // NOTE: slot_idx can be -1 if the token is padded
@@ -239,9 +239,9 @@ __global__ void reshape_and_cache_flash_kernel(
239 value_cache[tgt_key_value_idx] = tgt_value;
240 } else {
241 key_cache[tgt_key_value_idx] =
242 - fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, k_scale);
242 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, *k_scale);
243 value_cache[tgt_key_value_idx] =
244 - fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, v_scale);
244 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, *v_scale);
245 }
246 }
247 }
@@ -258,7 +258,9 @@ __global__ void reshape_and_cache_flash_kernel(
258 reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
259 reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
260 slot_mapping.data_ptr<int64_t>(), key_stride, value_stride, \
261 - num_heads, head_size, block_size, x, k_scale, v_scale);
261 + num_heads, head_size, block_size, x, \
262 + reinterpret_cast<const float*>(k_scale.data_ptr()), \
263 + reinterpret_cast<const float*>(v_scale.data_ptr()));
264
265 void reshape_and_cache(
266 torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -268,8 +270,8 @@ void reshape_and_cache(
270 torch::Tensor&
271 value_cache, // [num_blocks, num_heads, head_size, block_size]
272 torch::Tensor& slot_mapping, // [num_tokens]
271 - const std::string& kv_cache_dtype, const double k_scale,
272 - const double v_scale) {
273 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
274 + torch::Tensor& v_scale) {
275 int num_tokens = key.size(0);
276 int num_heads = key.size(1);
277 int head_size = key.size(2);
@@ -299,7 +301,9 @@ void reshape_and_cache(
301 reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
302 reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
303 slot_mapping.data_ptr<int64_t>(), block_stride, key_stride, \
302 - value_stride, num_heads, head_size, block_size, k_scale, v_scale);
304 + value_stride, num_heads, head_size, block_size, \
305 + reinterpret_cast<const float*>(k_scale.data_ptr()), \
306 + reinterpret_cast<const float*>(v_scale.data_ptr()));
307
308 void reshape_and_cache_flash(
309 torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -308,8 +312,8 @@ void reshape_and_cache_flash(
312 torch::Tensor&
313 value_cache, // [num_blocks, block_size, num_heads, head_size]
314 torch::Tensor& slot_mapping, // [num_tokens] or [num_actual_tokens]
311 - const std::string& kv_cache_dtype, const double k_scale,
312 - const double v_scale) {
315 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
316 + torch::Tensor& v_scale) {
317 // NOTE(woosuk): In vLLM V1, key.size(0) can be different from
318 // slot_mapping.size(0) because of padding for CUDA graphs.
319 // In vLLM V0, key.size(0) is always equal to slot_mapping.size(0) because
csrc/cpu/attention.cpp
+6 -6
@@ -460,11 +460,11 @@ void paged_attention_v1(
460 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
461 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
462 int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
463 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
464 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
463 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
464 + torch::Tensor& v_scale, const int64_t tp_rank,
465 + const int64_t blocksparse_local_blocks,
466 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
467 const int64_t blocksparse_head_sliding_step) {
467 - TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
468 TORCH_CHECK(blocksparse_vert_stride <= 1,
469 "CPU backend does not support blocksparse attention yet.");
470 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
@@ -782,11 +782,11 @@ void paged_attention_v2(
782 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
783 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
784 int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
785 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
786 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
785 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
786 + torch::Tensor& v_scale, const int64_t tp_rank,
787 + const int64_t blocksparse_local_blocks,
788 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
789 const int64_t blocksparse_head_sliding_step) {
789 - TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
790 TORCH_CHECK(blocksparse_vert_stride <= 1,
791 "CPU backend does not support blocksparse attention yet.");
792 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
csrc/cpu/cache.cpp
+2 -4
@@ -107,10 +107,8 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
107 void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
108 torch::Tensor& key_cache, torch::Tensor& value_cache,
109 torch::Tensor& slot_mapping,
110 - const std::string& kv_cache_dtype, double k_scale,
111 - double v_scale) {
112 - TORCH_CHECK(k_scale == 1.0f && v_scale == 1.0f);
113 -
110 + const std::string& kv_cache_dtype,
111 + torch::Tensor& k_scale, torch::Tensor& v_scale) {
112 int num_tokens = key.size(0);
113 int num_heads = key.size(1);
114 int head_size = key.size(2);
csrc/cpu/torch_bindings.cpp
+3 -3
@@ -30,7 +30,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
30 " Tensor value_cache, int num_kv_heads, float scale,"
31 " Tensor block_tables, Tensor seq_lens, int block_size,"
32 " int max_seq_len, Tensor? alibi_slopes,"
33 - " str kv_cache_dtype, float k_scale, float v_scale,"
33 + " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
34 " int tp_rank, int blocksparse_local_blocks,"
35 " int blocksparse_vert_stride, int blocksparse_block_size,"
36 " int blocksparse_head_sliding_step) -> ()");
@@ -44,7 +44,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
44 " Tensor value_cache, int num_kv_heads, float scale,"
45 " Tensor block_tables, Tensor seq_lens, int block_size,"
46 " int max_seq_len, Tensor? alibi_slopes,"
47 - " str kv_cache_dtype, float k_scale, float v_scale,"
47 + " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
48 " int tp_rank, int blocksparse_local_blocks,"
49 " int blocksparse_vert_stride, int blocksparse_block_size,"
50 " int blocksparse_head_sliding_step) -> ()");
@@ -148,7 +148,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
148 " Tensor! key_cache, Tensor! value_cache,"
149 " Tensor slot_mapping,"
150 " str kv_cache_dtype,"
151 - " float k_scale, float v_scale) -> ()");
151 + " Tensor k_scale, Tensor v_scale) -> ()");
152 cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
153 }
154
csrc/ops.h
+6 -4
@@ -34,8 +34,9 @@ void paged_attention_v1(
34 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
35 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
36 int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
37 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
38 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
37 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
38 + torch::Tensor& v_scale, const int64_t tp_rank,
39 + const int64_t blocksparse_local_blocks,
40 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
41 const int64_t blocksparse_head_sliding_step);
42
@@ -45,8 +46,9 @@ void paged_attention_v2(
46 torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
47 torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
48 int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
48 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
49 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
49 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
50 + torch::Tensor& v_scale, const int64_t tp_rank,
51 + const int64_t blocksparse_local_blocks,
52 const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
53 const int64_t blocksparse_head_sliding_step);
54
csrc/rocm/attention.cu
+10 -7
@@ -218,7 +218,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
218 scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
219 // head_size]
220 scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
221 - int max_ctx_blocks, float k_scale, float v_scale) {
221 + int max_ctx_blocks, const float* k_scale_ptr, const float* v_scale_ptr) {
222 constexpr int NWARPS = NUM_THREADS / WARP_SIZE;
223 const int warpid = threadIdx.x / WARP_SIZE;
224 const int laneid = threadIdx.x % WARP_SIZE;
@@ -406,7 +406,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
406 // Vlocalb8[h][b * BLOCK_SIZE / 8 + d] = v_ptrh8be[d];
407 const _B8x8 Vlocalb8 = v_ptrh8be[d];
408 Vlocal[h][b * BLOCK_SIZE / 8 + d] =
409 - scaled_convert_b8x8<scalar_t, KV_DTYPE>(Vlocalb8, v_scale);
409 + scaled_convert_b8x8<scalar_t, KV_DTYPE>(Vlocalb8, *v_scale_ptr);
410 }
411 }
412 }
@@ -416,7 +416,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
416 #pragma unroll
417 for (int d = 0; d < KHELOOP; d++) {
418 Klocal[d] =
419 - scaled_convert_b8x8<scalar_t, KV_DTYPE>(Klocalb8[d], k_scale);
419 + scaled_convert_b8x8<scalar_t, KV_DTYPE>(Klocalb8[d], *k_scale_ptr);
420 }
421 }
422
@@ -890,7 +890,7 @@ __global__ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_QKV_kernel(
890 scalar_t* __restrict__ out, // [num_seqs, num_heads, max_num_partitions,
891 // head_size]
892 scalar_t* __restrict__ final_out, // [num_seqs, num_heads, head_size]
893 - int max_ctx_blocks, float k_scale, float v_scale) {
893 + int max_ctx_blocks, const float* k_scale, const float* v_scale) {
894 UNREACHABLE_CODE
895 }
896
@@ -919,7 +919,7 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel(
919 block_tables_ptr, context_lens_ptr, max_num_blocks_per_seq, \
920 alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
921 exp_sums_ptr, max_logits_ptr, tmp_out_ptr, out_ptr, max_ctx_blocks, \
922 - k_scale, v_scale);
922 + k_scale_ptr, v_scale_ptr);
923
924 template <typename T, typename KVT, vllm::Fp8KVCacheDataType KV_DTYPE,
925 int BLOCK_SIZE, int HEAD_SIZE, int PARTITION_SIZE = 512>
@@ -929,7 +929,7 @@ void paged_attention_custom_launcher(
929 torch::Tensor& value_cache, const int num_kv_heads, float scale,
930 torch::Tensor& block_tables, torch::Tensor& context_lens,
931 int max_context_len, const std::optional<torch::Tensor>& alibi_slopes,
932 - float k_scale, float v_scale) {
932 + torch::Tensor& k_scale, torch::Tensor& v_scale) {
933 int num_seqs = query.size(0);
934 int num_heads = query.size(1);
935 int head_size = query.size(2);
@@ -953,6 +953,8 @@ void paged_attention_custom_launcher(
953 KVT* value_cache_ptr = reinterpret_cast<KVT*>(value_cache.data_ptr());
954 int* block_tables_ptr = block_tables.data_ptr<int>();
955 int* context_lens_ptr = context_lens.data_ptr<int>();
956 + const float* k_scale_ptr = reinterpret_cast<const float*>(k_scale.data_ptr());
957 + const float* v_scale_ptr = reinterpret_cast<const float*>(v_scale.data_ptr());
958
959 const int max_ctx_blocks = DIVIDE_ROUND_UP(max_context_len, BLOCK_SIZE);
960 const int max_num_partitions =
@@ -1087,7 +1089,8 @@ void paged_attention(
1089 torch::Tensor& context_lens, // [num_seqs]
1090 int64_t block_size, int64_t max_context_len,
1091 const std::optional<torch::Tensor>& alibi_slopes,
1090 - const std::string& kv_cache_dtype, double k_scale, double v_scale) {
1092 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
1093 + torch::Tensor& v_scale) {
1094 const int head_size = query.size(2);
1095 if (kv_cache_dtype == "auto") {
1096 if (query.dtype() == at::ScalarType::Half) {
csrc/rocm/ops.h
+2 -2
@@ -10,5 +10,5 @@ void paged_attention(torch::Tensor& out, torch::Tensor& exp_sums,
10 torch::Tensor& context_lens, int64_t block_size,
11 int64_t max_context_len,
12 const std::optional<torch::Tensor>& alibi_slopes,
13 - const std::string& kv_cache_dtype, double k_scale,
14 - double v_scale);
13 + const std::string& kv_cache_dtype, torch::Tensor& k_scale,
14 + torch::Tensor& v_scale);
csrc/rocm/torch_bindings.cpp
+1 -1
@@ -27,7 +27,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, rocm_ops) {
27 " int max_context_len,"
28 " Tensor? alibi_slopes,"
29 " str kv_cache_dtype,"
30 - " float k_scale, float v_scale) -> ()");
30 + " Tensor k_scale, Tensor v_scale) -> ()");
31 rocm_ops.impl("paged_attention", torch::kCUDA, &paged_attention);
32 }
33
csrc/torch_bindings.cpp
+4 -4
@@ -30,7 +30,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
30 " Tensor value_cache, int num_kv_heads, float scale,"
31 " Tensor block_tables, Tensor seq_lens, int block_size,"
32 " int max_seq_len, Tensor? alibi_slopes,"
33 - " str kv_cache_dtype, float k_scale, float v_scale,"
33 + " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
34 " int tp_rank, int blocksparse_local_blocks,"
35 " int blocksparse_vert_stride, int blocksparse_block_size,"
36 " int blocksparse_head_sliding_step) -> ()");
@@ -44,7 +44,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
44 " Tensor value_cache, int num_kv_heads, float scale,"
45 " Tensor block_tables, Tensor seq_lens, int block_size,"
46 " int max_seq_len, Tensor? alibi_slopes,"
47 - " str kv_cache_dtype, float k_scale, float v_scale,"
47 + " str kv_cache_dtype, Tensor k_scale, Tensor v_scale,"
48 " int tp_rank, int blocksparse_local_blocks,"
49 " int blocksparse_vert_stride, int blocksparse_block_size,"
50 " int blocksparse_head_sliding_step) -> ()");
@@ -449,7 +449,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
449 " Tensor! key_cache, Tensor! value_cache,"
450 " Tensor slot_mapping,"
451 " str kv_cache_dtype,"
452 - " float k_scale, float v_scale) -> ()");
452 + " Tensor k_scale, Tensor v_scale) -> ()");
453 cache_ops.impl("reshape_and_cache", torch::kCUDA, &reshape_and_cache);
454
455 // Reshape the key and value tensors and cache them.
@@ -459,7 +459,7 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
459 " Tensor! value_cache,"
460 " Tensor slot_mapping,"
461 " str kv_cache_dtype,"
462 - " float k_scale, float v_scale) -> ()");
462 + " Tensor k_scale, Tensor v_scale) -> ()");
463 cache_ops.impl("reshape_and_cache_flash", torch::kCUDA,
464 &reshape_and_cache_flash);
465
docs/source/features/quantization/quantized_kvcache.md
+6 -4
@@ -35,16 +35,18 @@ Studies have shown that FP8 E4M3 quantization typically only minimally degrades
35 Here is an example of how to enable FP8 quantization:
36
37 ```python
38 +# To calculate kv cache scales on the fly enable the calculate_kv_scales
39 +# parameter
40 +
41 from vllm import LLM, SamplingParams
42
43 sampling_params = SamplingParams(temperature=0.7, top_p=0.8)
41 -llm = LLM(model="meta-llama/Llama-3.1-8B-Instruct", kv_cache_dtype="fp8")
44 +llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",
45 + kv_cache_dtype="fp8",
46 + calculate_kv_scales=True)
47 prompt = "London is the capital of"
48 out = llm.generate(prompt, sampling_params)[0].outputs[0].text
49 print(out)
45 -
46 -# output w/ scaling factors: England, the United Kingdom, and one of the world's leading financial,
47 -# output w/o scaling factors: England, located in the southeastern part of the country. It is known
50 ```
51
52 The `kv_cache_dtype` argument specifies the data type for KV cache storage:
examples/other/fp8/README.md deleted
-96
@@ -1,96 +0,0 @@
1 -# FP8 KV Cache
2 -
3 -This utility extracts the KV cache scaling factors from a quantized HF (Hugging Face) model. The extracted scaling factors are saved to a JSON file, which can later be used by vLLM (variable-length language model) during runtime. This tool is particularly useful when the KV cache data type is FP8 and is intended for use on ROCm (AMD GPU) platforms.
4 -
5 -## Prerequisites
6 -
7 -- Python 3.x
8 -- PyTorch
9 -- NumPy
10 -- Hugging Face Transformers
11 -- Hugging Face Hub
12 -- AMMO
13 -
14 -Before incorporating the FP8 datatype for inference workloads, you must adhere to the following steps:
15 -1. Install all necessary prerequisites and dependencies.
16 -2. Convert HF model into a quantized HF model.
17 -3. Extract KV Cache Scaling Factors from quantized HF model.
18 -4. Load KV Cache Scaling Factors into VLLM.
19 -
20 -### 2. Convert HF model into a quantized HF model.
21 -Note: The following steps are adapted from the [TensorRT-LLM repository](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/quantization/README.md).
22 -
23 -`quantize.py` (examples/other/fp8/quantizer/quantize.py) uses the quantization toolkit (AMMO) to calibrate the PyTorch models and export TensorRT-LLM checkpoints. Each TensorRT-LLM checkpoint contains a config file (in .json format) and one or several rank weight files (in .safetensors format).
24 -
25 -The detailed quantization toolkit (AMMO) conversion guide for FP8 can be found at `examples/other/fp8/quantizer/README.md`.
26 -
27 -### 3. Extract KV Cache Scaling Factors from quantized HF model.
28 -`extract_scales.py` (examples/other/fp8/extract_scales.py) can be utilized to extract the KV cache scaling factors from your quantized HF model, however at the moment, this tool exclusively supports Llama 2 models. It is also important to note the following:
29 -1. **File Structure**: The utility operates under the assumption that all parameters, including KV cache scaling factors, corresponding to a particular Tensor Parallelism (TP) rank are stored in a single file. These files must adhere to a specific naming convention where the TP rank is immediately identified after a specific keyword (e.g., "rank") in the filename.
30 -
31 -2. **TP Decomposition**: The utility assumes consistency between the TP decomposition employed by the quantizer tool and that used by vLLM.
32 -
33 -3. **AMMO Compatibility**: Currently, the generated KV cache scaling factors for AMMO remain uniform across all TP ranks.
34 -
35 -```python
36 -# prerequisites:
37 -# - Quantized HF LLaMa 2 model
38 -python3 examples/other/fp8/extract_scales.py --help
39 -Usage: extract_scales.py [-h] --quantized_model QUANTIZED_MODEL [--load_format {auto,safetensors,npz,pt}] [--output_dir OUTPUT_DIR] [--output_name OUTPUT_NAME] [--tp_size TP_SIZE]
40 -
41 -KV Scale Extraction Example
42 -
43 -optional arguments:
44 ---quantized_model: Specify either the local path to, or name of, a quantized HF model. It is expected that the quantization format is FP8_E4M3, for use on ROCm (AMD GPU).
45 -Optional arguments:
46 ---cache_dir: Specify a cache directory to use in the event of a HF model download. (Default: None)
47 ---load_format: Specify the format of the model's tensor files containing the KV cache scaling factors. (Choices: auto, safetensors, npz, pt; Default: auto)
48 ---revision: Specify the model's revision number. (Default: None)
49 ---output_dir: Specify the output directory. By default the KV cache scaling factors will be saved in the model directory. (Default: None)
50 ---output_name: Specify the output filename. (Default: kv_cache_scales.json)
51 ---tp_size: Specify the tensor-parallel (TP) size that the quantized model should correspond to. If specified, during KV cache scaling factor extraction the observed TP size will be checked against this and an error will be raised if there is a mismatch. (Default: None)
52 -```
53 -```python
54 -Example:
55 -python3 examples/other/fp8/extract_scales.py --quantized_model <QUANTIZED_MODEL_DIR> --tp_size <TENSOR_PARALLEL_SIZE> --output_dir <PATH_TO_OUTPUT_DIR>
56 -```
57 -### 4. Load KV Cache Scaling Factors into VLLM.
58 -This script evaluates the inference throughput of language models using various backends such as vLLM. It measures the time taken to process a given number of prompts and generate sequences for each prompt. The recently generated KV cache scaling factors are now integrated into the benchmarking process and allow for KV cache scaling factors to be utilized for FP8.
59 -```
60 -# prerequisites:
61 -# - LLaMa 2 kv_cache_scales.json file
62 -
63 -python3 benchmarks/benchmark_throughput.py --help
64 -usage: benchmark_throughput.py [-h] [--backend {vllm,hf,mii}] [--dataset DATASET] [--input-len INPUT_LEN] [--output-len OUTPUT_LEN] [--model MODEL]
65 - [--tokenizer TOKENIZER] [--quantization {awq,gptq,None}] [--tensor-parallel-size TENSOR_PARALLEL_SIZE] [--n N]
66 - [--use-beam-search] [--num-prompts NUM_PROMPTS] [--seed SEED] [--hf-max-batch-size HF_MAX_BATCH_SIZE] [--trust-remote-code]
67 - [--max-model-len MAX_MODEL_LEN] [--dtype {auto,half,float16,bfloat16,float,float32}] [--enforce-eager] [--kv-cache-dtype {auto,fp8}]
68 - [--quantization-param-path KV_CACHE_quantization_param_path]
69 -
70 -Benchmark Throughput Example
71 -optional arguments:
72 - -h, --help show this help message and exit
73 - --backend {vllm,hf,mii}
74 - --dataset DATASET Path to the dataset.
75 - --input-len INPUT_LEN Input prompt length for each request
76 - --output-len OUTPUT_LEN Output length for each request. Overrides the output length from the dataset.
77 - --model MODEL
78 - --tokenizer TOKENIZER
79 - --quantization {awq,gptq,None}, -q {awq,gptq,None}
80 - --tensor-parallel-size TENSOR_PARALLEL_SIZE, -tp TENSOR_PARALLEL_SIZE
81 - --n N Number of generated sequences per prompt.
82 - --use-beam-search
83 - --num-prompts NUM_PROMPTS Number of prompts to process.
84 - --seed SEED
85 - --hf-max-batch-size HF_MAX_BATCH_SIZE Maximum batch size for HF backend.
86 - --trust-remote-code trust remote code from huggingface
87 - --max-model-len MAX_MODEL_LEN Maximum length of a sequence (including prompt and output). If None, will be derived from the model.
88 - --dtype {auto,half,float16,bfloat16,float,float32} data type for model weights and activations. The "auto" option will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.
89 - --enforce-eager enforce eager execution
90 - --kv-cache-dtype {auto,fp8} Data type for kv cache storage. If "auto", will use model data type. FP8_E5M2 (without scaling) is only supported on cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported ```for common inference criteria.
91 - --quantization-param-path QUANT_PARAM_JSON Path to the JSON file containing the KV cache scaling factors. This should generally be supplied, when KV cache dtype is FP8. Otherwise, KV cache scaling factors default to 1.0, which may cause accuracy issues. FP8_E5M2 (without scaling) is only supported on cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for common inference criteria.
92 -```
93 -Example:
94 -```console
95 -python3 benchmarks/benchmark_throughput.py --input-len <INPUT_LEN> --output-len <OUTPUT_LEN> -tp <TENSOR_PARALLEL_SIZE> --kv-cache-dtype fp8 --quantization-param-path <path/to/kv_cache_scales.json> --model <path-to-llama2>
96 -```
examples/other/fp8/extract_scales.py deleted
-367
@@ -1,367 +0,0 @@
1 -import argparse
2 -import glob
3 -import json
4 -import os
5 -from typing import Any, Callable, Dict, List, Optional, Tuple
6 -
7 -import numpy as np
8 -import torch
9 -from safetensors.torch import safe_open
10 -
11 -from vllm.model_executor.layers.quantization.schema import QuantParamSchema
12 -
13 -
14 -# Adapted from vllm/model_executor/model_loader/weight_utils.py
15 -# The main differences are that we add the NPZ format and simplify
16 -# its functionality drastically for our purposes (e.g. we assume that
17 -# the quantized model exists locally and there is no need to download it)
18 -def _prepare_hf_weights(
19 - quantized_model_dir: str,
20 - load_format: str = "auto",
21 - fall_back_to_pt: bool = True,
22 -) -> Tuple[List[str], bool]:
23 - if not os.path.isdir(quantized_model_dir):
24 - raise FileNotFoundError(
25 - f"The quantized model directory `{quantized_model_dir}` "
26 - "does not exist.")
27 - use_safetensors = False
28 - # Some quantized models use .pt files for storing the weights.
29 - if load_format == "auto":
30 - allow_patterns = ["*.safetensors", "*.bin"]
31 - elif load_format == "safetensors":
32 - use_safetensors = True
33 - allow_patterns = ["*.safetensors"]
34 - elif load_format == "pt":
35 - allow_patterns = ["*.pt"]
36 - elif load_format == "npz":
37 - allow_patterns = ["*.npz"]
38 - else:
39 - raise ValueError(f"Unknown load_format: {load_format}")
40 - if fall_back_to_pt:
41 - allow_patterns += ["*.pt"]
42 -
43 - hf_weights_files: List[str] = []
44 - for pattern in allow_patterns:
45 - hf_weights_files += glob.glob(
46 - os.path.join(quantized_model_dir, pattern))
47 - if len(hf_weights_files) > 0:
48 - if pattern == "*.safetensors":
49 - use_safetensors = True
50 - break
51 -
52 - if not use_safetensors:
53 - # Exclude files that are not needed for inference.
54 - # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
55 - blacklist = [
56 - "training_args.bin",
57 - "optimizer.bin",
58 - "optimizer.pt",
59 - "scheduler.pt",
60 - "scaler.pt",
61 - ]
62 - hf_weights_files = [
63 - f for f in hf_weights_files
64 - if not any(f.endswith(x) for x in blacklist)
65 - ]
66 -
67 - if len(hf_weights_files) == 0:
68 - raise RuntimeError(
69 - f"Cannot find any model weights with `{quantized_model_dir}`")
70 -
71 - return hf_weights_files, use_safetensors
72 -
73 -
74 -# Adapted from vllm/model_executor/model_loader/weight_utils.py
75 -def _hf_tensorfile_iterator(filename: str, load_format: str,
76 - use_safetensors: bool):
77 - if load_format == "npz":
78 - assert not use_safetensors
79 - with np.load(filename) as data:
80 - for name in data.files:
81 - param = torch.from_numpy(data[name])
82 - yield name, param
83 - elif use_safetensors:
84 - with safe_open(filename, framework="pt") as f:
85 - for name in f.keys(): # NOQA: SIM118
86 - param = f.get_tensor(name)
87 - yield name, param
88 - else:
89 - state = torch.load(filename, map_location="cpu")
90 - for name, param in state.items():
91 - yield name, param
92 - del state
93 - torch.cuda.empty_cache()
94 -
95 -
96 -def _kv_scales_extractor(
97 - hf_tensor_files: List[str],
98 - use_safetensors: bool,
99 - rank_keyword: str = "rank",
100 - expected_tp_size: Optional[int] = None) -> Dict[int, Dict[int, float]]:
101 - """
102 - Given a list of files containing tensor data, attempt to extract KV cache
103 - scales from these files. Intended as a helper function taking in the output
104 - from _prepare_hf_weights.
105 - Args:
106 - rank_keyword Matches the number immediately after this keyword in the
107 - tensor filename to determine the TP rank corresponding
108 - to said tensor file
109 - expected_tp_size If specified, the TP size of the tensor files is checked
110 - against this and an error is raised if they don't match.
111 - Returns a dictionary mapping TP ranks to their relevant KV cache scales.
112 - The per-rank scales are themselves represented as a dictionary of layer
113 - indices to the respective per-layer scale.
114 - """
115 - for char in rank_keyword:
116 - assert not char.isdecimal(
117 - ), f"Rank keyword {rank_keyword} contains a numeric character!"
118 - rank_scales_map: Dict[int, Dict[int, float]] = {}
119 - for tensor_file in hf_tensor_files:
120 - try:
121 - rank_idx = tensor_file.find(rank_keyword)
122 - if rank_idx != -1:
123 - start_idx = rank_idx + len(rank_keyword)
124 - stop_idx = start_idx
125 - while stop_idx < len(
126 - tensor_file) and tensor_file[stop_idx].isdecimal():
127 - stop_idx += 1
128 - if stop_idx == start_idx:
129 - raise RuntimeError("Did not find rank # in filename.")
130 - rank = int(tensor_file[start_idx:stop_idx])
131 - elif len(hf_tensor_files) == 1:
132 - # Since there is only one tensor file, we can assume
133 - # that it's intended for TP rank 0
134 - rank = 0
135 - else:
136 - raise RuntimeError(
137 - f"Filename does not contain '{rank_keyword}'.")
138 - except RuntimeError:
139 - print("Unable to determine TP rank "
140 - f"corresponding to file '{tensor_file}'")
141 - raise
142 -
143 - if rank not in rank_scales_map:
144 - layer_scales_map: Dict[int, float] = {}
145 - rank_scales_map[rank] = layer_scales_map
146 - else:
147 - raise RuntimeError(
148 - f"Tensor file '{tensor_file}' shares TP rank {rank} "
149 - "with another tensor file.")
150 -
151 - module_delimiter = ":" if args.load_format == "npz" else "."
152 - for name, param in _hf_tensorfile_iterator(tensor_file,
153 - args.load_format,
154 - use_safetensors):
155 - if "kv_cache_scaling_factor" in name:
156 - nums = [
157 - int(s) for s in name.split(module_delimiter)
158 - if s.isdecimal()
159 - ]
160 - assert len(
161 - nums) == 1, f"Could not determine layer idx for {name}"
162 - layer_idx = nums[0]
163 - assert layer_idx not in layer_scales_map, f"Duplicate scaling"\
164 - f" factor corresponding to layer {layer_idx}"
165 - try:
166 - layer_scales_map[layer_idx] = param.item()
167 - except RuntimeError:
168 - print(
169 - "This utility supports only per-tensor scalar scales "
170 - f"for now. The tensor\n {name} = {param} \nis an "
171 - "invalid scale factor.")
172 - raise
173 -
174 - if all(
175 - len(layer_scales_map) == 0
176 - for layer_scales_map in rank_scales_map.values()):
177 - # Note: this is true even if the rank_scales_map is empty
178 - print("WARNING: No KV cache scale factors found. No output saved.")
179 - return None
180 - empirical_tp_world_size = max(rank_scales_map.keys()) + 1
181 - if expected_tp_size is not None:
182 - assert expected_tp_size == empirical_tp_world_size, \
183 - f"User expected TP world size = {expected_tp_size} " \
184 - "from model but tool is expecting TP world size = " \
185 - f"{empirical_tp_world_size} from model instead."
186 - for i in range(empirical_tp_world_size):
187 - assert i in rank_scales_map, "Expected TP world size = "\
188 - f"{empirical_tp_world_size} but did not find KV " \
189 - f"cache scaling factors for TP rank {i}"
190 - print(f"Found TP world size = {empirical_tp_world_size} "
191 - "when extracting KV cache scales!")
192 - return rank_scales_map
193 -
194 -
195 -def _metadata_extractor(quantized_model_dir: str,
196 - metadata_extract_fns: \
197 - Dict[str, Callable[[Dict[str, Any]], Any]]) \
198 - -> Dict[str, Any]:
199 - """
200 - Given a directory containing quantized model files, this function
201 - aims to extract metadata from the JSON files within this directory.
202 - Each JSON file is expected to represent a dictionary in JSON
203 - format (referred to as a "JSON-dictionary"). Metadata extraction is
204 - defined by a dictionary called metadata_extract_fns, where each
205 - metadata field name is mapped to an extraction function.
206 -
207 - These extraction functions are designed to take a JSON-dictionary
208 - as their only argument and return the corresponding metadata.
209 - While extraction functions are permitted to raise exceptions, they
210 - should only raise a KeyError or ValueError if the metadata field
211 - cannot be extracted from the current JSON-dictionary, yet there's
212 - a possibility of finding it in another JSON-dictionary.
213 -
214 - The function returns a dictionary that maps metadata fields to
215 - their extracted data. The keys of this dictionary correspond exactly
216 - to those in metadata_extract_fns. If any fields fail to be extracted,
217 - their corresponding values are set to None, and a warning is printed.
218 - """
219 - if not os.path.isdir(quantized_model_dir):
220 - raise FileNotFoundError(
221 - f"The quantized model directory `{quantized_model_dir}` "
222 - "does not exist.")
223 - metadata_files = glob.glob(os.path.join(quantized_model_dir, "*.json"))
224 -
225 - result: Dict[str, Any] = {}
226 - for file in metadata_files:
227 - with open(file) as f:
228 - try:
229 - metadata = json.load(f)
230 - except json.JSONDecodeError:
231 - print(f"Could not parse `{file}` as a valid metadata file,"
232 - " skipping it.")
233 - continue
234 - if not isinstance(metadata, dict):
235 - print(f"The file `{file}` does not correspond to a "
236 - "JSON-serialized dictionary, skipping it.")
237 - continue
238 - for metadata_name, extract_fn in metadata_extract_fns.items():
239 - try:
240 - metadata_info = extract_fn(metadata)
241 - if metadata_name not in result:
242 - result[metadata_name] = metadata_info
243 - elif metadata_info != result[metadata_name]:
244 - raise RuntimeError(
245 - "Metadata mismatch! Originally found "
246 - f"{metadata_name} = {result[metadata_name]} but "
247 - f"now found {metadata_name} = {metadata_info} in "
248 - f"`{file}`")
249 - except KeyError:
250 - # It is possible that a given file does not contain some
251 - # of our selected metadata as it could be located in some
252 - # other metadata file.
253 - # 'EFINAE': extract_fn failure is not an error.
254 - pass
255 - except ValueError:
256 - # See above.
257 - pass
258 -
259 - # Warn if we cannot find any of the requested metadata
260 - for metadata_name in metadata_extract_fns:
261 - if metadata_name not in result:
262 - print("WARNING: Unable to find requested metadata field "
263 - f"`{metadata_name}`, setting it to None.")
264 - result[metadata_name] = None
265 -
266 - return result
267 -
268 -
269 -def main(args):
270 - metadata_extract_fns = {
271 - "model_type": lambda json_dict: json_dict["layers"][0]["decoder_type"],
272 - "tp_size": lambda json_dict: int(json_dict["tensor_parallel"]),
273 - "model_dtype": lambda json_dict: json_dict["dtype"]
274 - }
275 - recovered_metadata = _metadata_extractor(args.quantized_model,
276 - metadata_extract_fns)
277 - if args.tp_size is not None:
278 - metadata_tp_size = recovered_metadata["tp_size"]
279 - if metadata_tp_size is not None:
280 - assert args.tp_size == metadata_tp_size, \
281 - f"User expected TP world size = {args.tp_size} " \
282 - f"but found TP world size = {metadata_tp_size} from metadata!"
283 - expected_tp_size = args.tp_size or recovered_metadata["tp_size"]
284 - rank_keyword = "rank"
285 - hf_tensor_files, use_safetensors = _prepare_hf_weights(
286 - args.quantized_model, args.load_format)
287 - rank_scales_map = _kv_scales_extractor(hf_tensor_files, use_safetensors,
288 - rank_keyword, expected_tp_size)
289 - # Postprocess: formatting to the current schema. Consider pulling it
290 - # out into a dedicated function should it ever become more complicated.
291 - rank_scales_map = {
292 - rank: {k: scale[k]
293 - for k in sorted(scale.keys())}
294 - for rank, scale in rank_scales_map.items()
295 - }
296 - # TODO: Expand this with activation and weights scaling factors when
297 - # they are used in the future
298 - schema = QuantParamSchema(
299 - model_type=recovered_metadata["model_type"],
300 - kv_cache={
301 - "dtype": ("float8_e4m3fn" if len(rank_scales_map) > 0 else
302 - recovered_metadata["model_dtype"]),
303 - "scaling_factor":
304 - rank_scales_map
305 - },
306 - )
307 -
308 - if args.output_dir is None:
309 - output_file = os.path.join(args.quantized_model, args.output_name)
310 - else:
311 - if not os.path.isdir(args.output_dir):
312 - os.makedirs(args.output_dir, exist_ok=True)
313 - output_file = os.path.join(args.output_dir, args.output_name)
314 -
315 - with open(output_file, 'w') as f:
316 - f.write(schema.model_dump_json(indent=4))
317 - print(f"Completed! KV cache scaling factors saved to {output_file}")
318 -
319 -
320 -if __name__ == "__main__":
321 - parser = argparse.ArgumentParser(
322 - description="This simple utility extracts the "
323 - "KV cache scaling factors from a quantized HF model "
324 - "and saves them to a JSON file compatible with later "
325 - "use by vLLM (pass this file to the appropriate "
326 - "runtime typically using the argument "
327 - "--quantization-param-path <filename>). This is only used "
328 - "if the KV cache dtype is FP8 and on ROCm (AMD GPU).")
329 - parser.add_argument(
330 - "--quantized-model",
331 - help="Specify the directory containing a single quantized HF model. "
332 - "It is expected that the quantization format is FP8_E4M3, for use "
333 - "on ROCm (AMD GPU).",
334 - required=True)
335 - parser.add_argument(
336 - "--load_format",
337 - help="Optionally specify the format of the model's tensor files "
338 - "containing the KV cache scaling factors.",
339 - choices=["auto", "safetensors", "npz", "pt"],
340 - default="auto")
341 - parser.add_argument(
342 - "--output-dir",
343 - help="Optionally specify the output directory. By default the "
344 - "KV cache scaling factors will be saved in the model directory, "
345 - "however you can override this behavior here.",
346 - default=None)
347 - parser.add_argument(
348 - "--output-name",
349 - help="Optionally specify the output filename.",
350 - # TODO: Change this once additional scaling factors are enabled
351 - default="kv_cache_scales.json")
352 - parser.add_argument(
353 - "--tp-size",
354 - help="Optionally specify the tensor-parallel (TP) size that the "
355 - "quantized model should correspond to. If specified, during KV "
356 - "cache scaling factor extraction the observed TP size will be "
357 - "checked against this and an error will be raised if there is "
358 - "a mismatch. If not specified, the quantized model's expected "
359 - "TP size is instead inferred from the largest TP rank observed. "
360 - "The expected TP size is cross-checked against the TP ranks "
361 - "observed in the quantized model and an error is raised if any "
362 - "discrepancies are found.",
363 - default=None,
364 - type=int)
365 - args = parser.parse_args()
366 -
367 - main(args)
examples/other/fp8/quantizer/README.md deleted
-32
@@ -1,32 +0,0 @@
1 -### Quantizer Utilities
2 -`quantize.py`: NVIDIA Quantization utilities using TensorRT-Model-Optimizer, ported
3 -from TensorRT-LLM: [`examples/quantization/quantize.py`](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/quantization/quantize.py)
4 -
5 -### Prerequisite
6 -
7 -#### AMMO (AlgorithMic Model Optimization) Installation: nvidia-ammo 0.7.1 or later
8 -`pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com nvidia-ammo`
9 -
10 -#### AMMO Download (code and docs)
11 -`https://developer.nvidia.com/downloads/assets/cuda/files/nvidia-ammo/nvidia_ammo-0.5.0.tar.gz`
12 -`https://developer.nvidia.com/downloads/assets/cuda/files/nvidia-ammo/nvidia_ammo-0.7.1.tar.gz`
13 -
14 -### Usage
15 -
16 -#### Run on H100 system for speed if FP8; number of GPUs depends on the model size
17 -
18 -#### Example: quantize Llama2-7b model from HF to FP8 with FP8 KV Cache:
19 -`python quantize.py --model-dir ./ll2-7b --dtype float16 --qformat fp8 --kv-cache-dtype fp8 --output-dir ./ll2_7b_fp8 --calib-size 512 --tp-size 1`
20 -
21 -Outputs: model structure, quantized model & parameters (with scaling factors) are in JSON and Safetensors (npz is generated only for the reference)
22 -```
23 -# ll ./ll2_7b_fp8/
24 -total 19998244
25 -drwxr-xr-x 2 root root 4096 Feb 7 01:08 ./
26 -drwxrwxr-x 8 1060 1061 4096 Feb 7 01:08 ../
27 --rw-r--r-- 1 root root 176411 Feb 7 01:08 llama_tp1.json
28 --rw-r--r-- 1 root root 13477087480 Feb 7 01:09 llama_tp1_rank0.npz
29 --rw-r--r-- 1 root root 7000893272 Feb 7 01:08 rank0.safetensors
30 -#
31 -```
32 -
examples/other/fp8/quantizer/quantize.py deleted
-367
@@ -1,367 +0,0 @@
1 -# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # noqa: E501
2 -# SPDX-License-Identifier: Apache-2.0
3 -#
4 -# Licensed under the Apache License, Version 2.0 (the "License");
5 -# you may not use this file except in compliance with the License.
6 -# You may obtain a copy of the License at
7 -#
8 -# http://www.apache.org/licenses/LICENSE-2.0
9 -#
10 -# Unless required by applicable law or agreed to in writing, software
11 -# distributed under the License is distributed on an "AS IS" BASIS,
12 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 -# See the License for the specific language governing permissions and
14 -# limitations under the License.
15 -"""
16 -Adapted from examples/quantization/hf_ptq.py
17 -"""
18 -
19 -import argparse
20 -import copy
21 -import json
22 -import random
23 -import time
24 -
25 -import ammo.torch.quantization as atq
26 -import numpy as np
27 -import torch
28 -from ammo.torch.export import export_model_config
29 -from datasets import load_dataset
30 -from torch.utils.data import DataLoader
31 -from transformers import AutoModelForCausalLM, AutoTokenizer
32 -
33 -RAND_SEED = 1234
34 -MAX_SEQ_LEN = 2048
35 -
36 -EMPTY_CFG = {
37 - "quant_cfg": {
38 - "*weight_quantizer": {
39 - "enable": False,
40 - },
41 - "*input_quantizer": {
42 - "enable": False
43 - },
44 - "*lm_head*": {
45 - "enable": False
46 - },
47 - "*output_layer*": {
48 - "enable": False
49 - },
50 - "default": {
51 - "enable": False
52 - },
53 - },
54 - "algorithm": "max",
55 -}
56 -
57 -KV_CACHE_CFG = {
58 - "*.query_key_value.output_quantizer": {
59 - "num_bits": 8,
60 - "axis": None,
61 - "enable": True
62 - },
63 - "*.Wqkv.output_quantizer": {
64 - "num_bits": 8,
65 - "axis": None,
66 - "enable": True
67 - },
68 - "*.W_pack.output_quantizer": {
69 - "num_bits": 8,
70 - "axis": None,
71 - "enable": True
72 - },
73 - "*.c_attn.output_quantizer": {
74 - "num_bits": 8,
75 - "axis": None,
76 - "enable": True
77 - },
78 - "*.k_proj.output_quantizer": {
79 - "num_bits": 8,
80 - "axis": None,
81 - "enable": True
82 - },
83 - "*.v_proj.output_quantizer": {
84 - "num_bits": 8,
85 - "axis": None,
86 - "enable": True
87 - },
88 -}
89 -
90 -QUANT_CFG_CHOICES = {
91 - "int8_sq": atq.INT8_SMOOTHQUANT_CFG,
92 - "fp8": atq.FP8_DEFAULT_CFG,
93 - "int4_awq": atq.INT4_AWQ_CFG,
94 - "w4a8_awq": atq.W4A8_AWQ_BETA_CFG,
95 - "int8_wo": EMPTY_CFG,
96 - "int4_wo": EMPTY_CFG,
97 - "full_prec": EMPTY_CFG,
98 -}
99 -
100 -MODEL_NAME_PATTERN_MAP = {
101 - "GPT2": "gpt2",
102 - "Xverse": "llama",
103 - "Llama": "llama",
104 - "Mistral": "llama",
105 - "GPTJ": "gptj",
106 - "FalconForCausalLM": "falcon",
107 - "RWForCausalLM": "falcon",
108 - "baichuan": "baichuan",
109 - "MPT": "mpt",
110 - "Bloom": "bloom",
111 - "ChatGLM": "chatglm",
112 - "QWen": "qwen",
113 -}
114 -
115 -
116 -def get_tokenizer(ckpt_path, max_seq_len=MAX_SEQ_LEN, model_type=None):
117 - print(f"Initializing tokenizer from {ckpt_path}")
118 - tokenizer = AutoTokenizer.from_pretrained(
119 - ckpt_path,
120 - model_max_length=max_seq_len,
121 - padding_side="left",
122 - trust_remote_code=True,
123 - )
124 - if model_type and model_type == "qwen":
125 - # qwen use token id 151643 as pad and eos tokens
126 - tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
127 - tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
128 -
129 - # can't set attribute 'pad_token' for "<unk>"
130 - if tokenizer.pad_token != "<unk>":
131 - tokenizer.pad_token = tokenizer.eos_token
132 - if tokenizer.pad_token is None:
133 - tokenizer.pad_token = tokenizer.eos_token
134 - assert (tokenizer.pad_token
135 - is not None), f"Pad token for {model_type} cannot be set!"
136 -
137 - return tokenizer
138 -
139 -
140 -def get_model(ckpt_path, dtype="fp16", device="cuda"):
141 - print(f"Initializing model from {ckpt_path}")
142 - if dtype == "bf16" or dtype == "bfloat16":
143 - dtype = torch.bfloat16
144 - elif dtype == "fp16" or dtype == "float16":
145 - dtype = torch.float16
146 - elif dtype == "fp32" or dtype == "float32":
147 - dtype = torch.float32
148 - else:
149 - raise NotImplementedError(f"Unknown dtype {dtype}")
150 -
151 - # model_kwargs = {"torch_dtype": dtype}
152 - model_kwargs = {"torch_dtype": "auto"}
153 -
154 - model = AutoModelForCausalLM.from_pretrained(ckpt_path,
155 - device_map="auto",
156 - **model_kwargs,
157 - trust_remote_code=True)
158 - model.eval()
159 -
160 - model_dtype = next(model.parameters()).dtype
161 - if dtype != model_dtype:
162 - print("[TensorRT-LLM][WARNING] The manually set model data type is "
163 - f"{dtype}, but the data type of the HuggingFace model is "
164 - f"{model_dtype}.")
165 -
166 - return model
167 -
168 -
169 -def get_model_type(model):
170 - for k, v in MODEL_NAME_PATTERN_MAP.items():
171 - if k.lower() in type(model).__name__.lower():
172 - return v
173 - return None
174 -
175 -
176 -def get_calib_dataloader(data="cnn_dailymail",
177 - tokenizer=None,
178 - batch_size=1,
179 - calib_size=512,
180 - block_size=512,
181 - device=None):
182 - print("Loading calibration dataset")
183 - if data == "pileval":
184 - dataset = load_dataset(
185 - "json",
186 - data_files="https://the-eye.eu/public/AI/pile/val.jsonl.zst",
187 - split="train")
188 - dataset = dataset["text"][:calib_size]
189 - elif data == "cnn_dailymail":
190 - dataset = load_dataset("cnn_dailymail", name="3.0.0", split="train")
191 - dataset = dataset["article"][:calib_size]
192 - else:
193 - raise NotImplementedError
194 -
195 - batch_encoded = tokenizer.batch_encode_plus(dataset,
196 - return_tensors="pt",
197 - padding="max_length",
198 - truncation=True,
199 - max_length=block_size)
200 - if device:
201 - batch_encoded = batch_encoded.to(device)
202 - batch_encoded = batch_encoded["input_ids"]
203 -
204 - calib_dataloader = DataLoader(batch_encoded,
205 - batch_size=batch_size,
206 - shuffle=False)
207 -
208 - return calib_dataloader
209 -
210 -
211 -def quantize_model(model, quant_cfg, calib_dataloader=None):
212 -
213 - def calibrate_loop():
214 - if calib_dataloader is None:
215 - return
216 - """Adjusts weights and scaling factors based on selected algorithms."""
217 - for idx, data in enumerate(calib_dataloader):
218 - print(f"Calibrating batch {idx}")
219 - model(data)
220 -
221 - print("Starting quantization...")
222 - start_time = time.time()
223 - atq.quantize(model, quant_cfg, forward_loop=calibrate_loop)
224 - end_time = time.time()
225 - print("Quantization done. Total time used: {:.2f} s.".format(end_time -
226 - start_time))
227 -
228 - return model
229 -
230 -
231 -def main(args):
232 - if not torch.cuda.is_available():
233 - raise OSError("GPU is required for inference.")
234 -
235 - random.seed(RAND_SEED)
236 - np.random.seed(RAND_SEED)
237 -
238 - model = get_model(args.model_dir, args.dtype, args.device)
239 - model_type = get_model_type(model)
240 - tokenizer = get_tokenizer(args.model_dir, model_type=model_type)
241 -
242 - if args.qformat in ["full_prec", "int8_wo", "int4_wo"
243 - ] and args.kv_cache_dtype is None:
244 - print(f"No quantization applied, export {args.dtype} model")
245 - else:
246 - if "awq" in args.qformat:
247 - if args.calib_size > 32:
248 - print("AWQ calibration could take longer with calib_size = "
249 - f"{args.calib_size}, Using calib_size=32 instead")
250 - args.calib_size = 32
251 - print("\nAWQ calibration could take longer than other calibration "
252 - "methods. Please increase the batch size to speed up the "
253 - "calibration process. Batch size can be set by adding the "
254 - "argument --batch_size <batch_size> to the command line.\n")
255 -
256 - calib_dataloader = get_calib_dataloader(
257 - tokenizer=tokenizer,
258 - batch_size=args.batch_size,
259 - calib_size=args.calib_size,
260 - device=args.device,
261 - )
262 -
263 - if args.qformat in QUANT_CFG_CHOICES:
264 - quant_cfg = QUANT_CFG_CHOICES[args.qformat]
265 - else:
266 - raise ValueError(
267 - f"Unsupported quantization format: {args.qformat}")
268 -
269 - if "awq" in args.qformat:
270 - quant_cfg = copy.deepcopy(QUANT_CFG_CHOICES[args.qformat])
271 - weight_quantizer = quant_cfg["quant_cfg"][
272 - "*weight_quantizer"] # type: ignore
273 - if isinstance(weight_quantizer, list):
274 - weight_quantizer = weight_quantizer[0]
275 - weight_quantizer["block_sizes"][-1] = args.awq_block_size
276 -
277 - if args.kv_cache_dtype is not None:
278 - if args.kv_cache_dtype == "fp8":
279 - for value in KV_CACHE_CFG.values():
280 - value.update({"num_bits": (4, 3)}) # type: ignore
281 - quant_cfg["quant_cfg"].update(KV_CACHE_CFG) # type: ignore
282 -
283 - print(quant_cfg)
284 -
285 - model = quantize_model(model, quant_cfg, calib_dataloader)
286 -
287 - with torch.inference_mode():
288 - if model_type is None:
289 - print(f"Unknown model type {type(model).__name__}. Continue "
290 - "exporting...")
291 - model_type = f"unknown:{type(model).__name__}"
292 -
293 - export_path = args.output_dir
294 - start_time = time.time()
295 -
296 - if args.qformat == "int4_awq" and model_type == "qwen":
297 - torch.save(model.state_dict(), export_path)
298 - else:
299 - export_npz = (model_type not in [
300 - 'gptj', 'falcon', 'chatglm', 'mpt', 'llama', 'baichuan'
301 - ])
302 -
303 - # export safetensors
304 - export_model_config(
305 - model,
306 - model_type,
307 - getattr(torch, args.dtype),
308 - export_dir=export_path,
309 - inference_tensor_parallel=args.tp_size,
310 - inference_pipeline_parallel=args.pp_size,
311 - # export_tensorrt_llm_config=(not export_npz),
312 - export_tensorrt_llm_config=False,
313 - export_npz=export_npz)
314 -
315 - # Workaround for wo quantization
316 - if args.qformat in ["int8_wo", "int4_wo", "full_prec"]:
317 - with open(f"{export_path}/config.json") as f:
318 - tensorrt_llm_config = json.load(f)
319 - if args.qformat == "int8_wo":
320 - tensorrt_llm_config["quantization"]["quant_algo"] = 'W8A16'
321 - elif args.qformat == "int4_wo":
322 - tensorrt_llm_config["quantization"]["quant_algo"] = 'W4A16'
323 - else:
324 - tensorrt_llm_config["quantization"]["quant_algo"] = None
325 - with open(f"{export_path}/config.json", "w") as f:
326 - json.dump(tensorrt_llm_config, f, indent=4)
327 -
328 - end_time = time.time()
329 - print("Quantized model exported to {} \nTotal time used {:.2f} s.".
330 - format(export_path, end_time - start_time))
331 -
332 -
333 -if __name__ == "__main__":
334 - parser = argparse.ArgumentParser(description=__doc__)
335 - parser.add_argument("--model-dir",
336 - help="Specify where the HuggingFace model is",
337 - required=True)
338 - parser.add_argument("--device", default="cuda")
339 - parser.add_argument("--dtype", help="Model data type.", default="float16")
340 - parser.add_argument(
341 - "--qformat",
342 - help="Quantization format.",
343 - default="full_prec",
344 - choices=[
345 - "fp8", "int8_sq", "int4_awq", "w4a8_awq", "int8_wo", "int4_wo",
346 - "full_prec"
347 - ],
348 - )
349 - parser.add_argument("--batch-size",
350 - help="Batch size for calibration.",
351 - type=int,
352 - default=1)
353 - parser.add_argument("--calib-size",
354 - help="Number of samples for calibration.",
355 - type=int,
356 - default=512)
357 - parser.add_argument("--output-dir", default="exported_model")
358 - parser.add_argument("--tp-size", type=int, default=1)
359 - parser.add_argument("--pp-size", type=int, default=1)
360 - parser.add_argument("--awq-block-size", type=int, default=128)
361 - parser.add_argument("--kv-cache-dtype",
362 - help="KV Cache dtype.",
363 - default=None,
364 - choices=["int8", "fp8", None])
365 - args = parser.parse_args()
366 -
367 - main(args)
tests/fp8_kv/llama2-70b-fp8-kv/kv_cache_scales.json deleted
-90
@@ -1,90 +0,0 @@
1 -{
2 - "model_type": "llama",
3 - "kv_cache": {
4 - "dtype": "float8_e4m3fn",
5 - "scaling_factor": {
6 - "0": {
7 - "0": 0.0230364128947258,
8 - "1": 0.01979283057153225,
9 - "2": 0.0241350457072258,
10 - "3": 0.0308314748108387,
11 - "4": 0.0430733822286129,
12 - "5": 0.0370396226644516,
13 - "6": 0.0306222103536129,
14 - "7": 0.0357491634786129,
15 - "8": 0.0358189195394516,
16 - "9": 0.0443289652466774,
17 - "10": 0.0433175228536129,
18 - "11": 0.0416782945394516,
19 - "12": 0.0366908498108387,
20 - "13": 0.0432477705180645,
21 - "14": 0.0410505048930645,
22 - "15": 0.0457589291036129,
23 - "16": 0.0418526791036129,
24 - "17": 0.0432477705180645,
25 - "18": 0.0469447560608387,
26 - "19": 0.0514787957072258,
27 - "20": 0.0541294664144516,
28 - "21": 0.0587681382894516,
29 - "22": 0.0625,
30 - "23": 0.0585588738322258,
31 - "24": 0.0600237175822258,
32 - "25": 0.0588030144572258,
33 - "26": 0.0531180277466774,
34 - "27": 0.06396484375,
35 - "28": 0.0603027381002903,
36 - "29": 0.0582101047039032,
37 - "30": 0.0625348836183548,
38 - "31": 0.0585588738322258,
39 - "32": 0.0582798570394516,
40 - "33": 0.0575125589966774,
41 - "34": 0.0590820349752903,
42 - "35": 0.0614188089966774,
43 - "36": 0.0631975457072258,
44 - "37": 0.0615931935608387,
45 - "38": 0.0601283498108387,
46 - "39": 0.0571986623108387,
47 - "40": 0.0670340433716774,
48 - "41": 0.0523507259786129,
49 - "42": 0.0547223798930645,
50 - "43": 0.0631975457072258,
51 - "44": 0.0663713738322258,
52 - "45": 0.0603376142680645,
53 - "46": 0.0652204304933548,
54 - "47": 0.0734514519572258,
55 - "48": 0.0693708211183548,
56 - "49": 0.0725446492433548,
57 - "50": 0.0627790242433548,
58 - "51": 0.0691266804933548,
59 - "52": 0.0688825398683548,
60 - "53": 0.068429134786129,
61 - "54": 0.0605119988322258,
62 - "55": 0.0799386203289032,
63 - "56": 0.0853097140789032,
64 - "57": 0.0661969929933548,
65 - "58": 0.0689871683716774,
66 - "59": 0.0724051371216774,
67 - "60": 0.0541643425822258,
68 - "61": 0.0626743882894516,
69 - "62": 0.0628487765789032,
70 - "63": 0.0607212632894516,
71 - "64": 0.0589076466858387,
72 - "65": 0.0451660193502903,
73 - "66": 0.0453055277466774,
74 - "67": 0.0414341539144516,
75 - "68": 0.0385044664144516,
76 - "69": 0.0414341539144516,
77 - "70": 0.0466308631002903,
78 - "71": 0.0399693101644516,
79 - "72": 0.0437011756002903,
80 - "73": 0.0434221550822258,
81 - "74": 0.0428989976644516,
82 - "75": 0.0401785746216774,
83 - "76": 0.0431082621216774,
84 - "77": 0.0484444759786129,
85 - "78": 0.0417829267680645,
86 - "79": 0.0418178029358387
87 - }
88 - }
89 - }
90 -}
\ No newline at end of file
tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json deleted
-42
@@ -1,42 +0,0 @@
1 -{
2 - "model_type": "llama",
3 - "kv_cache": {
4 - "dtype": "float8_e4m3fn",
5 - "scaling_factor": {
6 - "0": {
7 - "0": 0.0152239128947258,
8 - "1": 0.0188860222697258,
9 - "2": 0.0354178324341774,
10 - "3": 0.0376674123108387,
11 - "4": 0.0418526791036129,
12 - "5": 0.0433175228536129,
13 - "6": 0.0397600457072258,
14 - "7": 0.0424455925822258,
15 - "8": 0.0415387861430645,
16 - "9": 0.0408412404358387,
17 - "10": 0.0395856611430645,
18 - "11": 0.0377371683716774,
19 - "12": 0.0400739423930645,
20 - "13": 0.040771484375,
21 - "14": 0.0393415205180645,
22 - "15": 0.0369001142680645,
23 - "16": 0.03857421875,
24 - "17": 0.0387486070394516,
25 - "18": 0.0403180830180645,
26 - "19": 0.0396205373108387,
27 - "20": 0.0375627800822258,
28 - "21": 0.0407366082072258,
29 - "22": 0.0432477705180645,
30 - "23": 0.0377022884786129,
31 - "24": 0.0399693101644516,
32 - "25": 0.0374581478536129,
33 - "26": 0.0413295216858387,
34 - "27": 0.0442243330180645,
35 - "28": 0.0424804724752903,
36 - "29": 0.0456891767680645,
37 - "30": 0.0409109964966774,
38 - "31": 0.0482352152466774
39 - }
40 - }
41 - }
42 -}
tests/kernels/test_attention.py
+1 -1
@@ -182,7 +182,7 @@ def test_paged_attention(
182 key_cache, value_cache = key_caches[0], value_caches[0]
183
184 # Using default kv_scale
185 - k_scale = v_scale = 1.0
185 + k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
186
187 # Call the paged attention kernel.
188 output = torch.empty_like(query)
tests/kernels/test_blocksparse_attention.py
+1 -1
@@ -210,7 +210,7 @@ def test_paged_attention(
210 key_cache, value_cache = key_caches[0], value_caches[0]
211
212 # Using default kv_scale
213 - k_scale = v_scale = 1.0
213 + k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
214 tp_rank = 0
215
216 # Call the paged attention kernel.
tests/kernels/test_cache.py
+5 -5
@@ -160,7 +160,7 @@ def test_reshape_and_cache(
160 cloned_value_cache = value_cache.clone()
161
162 # Using default kv_scale
163 - k_scale = v_scale = 1.0
163 + k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
164
165 # Call the reshape_and_cache kernel.
166 opcheck(torch.ops._C_cache_ops.reshape_and_cache,
@@ -258,8 +258,8 @@ def test_reshape_and_cache_flash(
258 del key_caches
259 del value_caches
260
261 - k_scale = key.amax().item() / 256
262 - v_scale = value.amax().item() / 256
261 + k_scale = (key.amax() / 256.0).to(torch.float32)
262 + v_scale = (value.amax() / 256.0).to(torch.float32)
263
264 # Clone the KV caches.
265 if kv_cache_dtype == "fp8":
@@ -284,12 +284,12 @@ def test_reshape_and_cache_flash(
284 result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
285 ops.convert_fp8(result_key_cache,
286 key_cache,
287 - k_scale,
287 + k_scale.item(),
288 kv_dtype=kv_cache_dtype)
289 result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
290 ops.convert_fp8(result_value_cache,
291 value_cache,
292 - v_scale,
292 + v_scale.item(),
293 kv_dtype=kv_cache_dtype)
294
295 # Run the reference implementation.
tests/kernels/test_prefix_prefill.py
+10
@@ -138,6 +138,7 @@ def test_contexted_kv_attention(
138 # to V_cache[num_blocks, num_kv_heads, head_size, block_size]
139 v_cache = v_cache.view(-1, block_size, num_kv_heads,
140 head_size).permute(0, 2, 3, 1).contiguous()
141 + k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
142
143 # Warm up the Triton kernel by calling it once before actually measuring
144 # generation time
@@ -153,6 +154,8 @@ def test_contexted_kv_attention(
154 b_seq_len,
155 b_ctx_len,
156 max_input_len,
157 + k_scale,
158 + v_scale,
159 sliding_window=sliding_window)
160 torch.cuda.synchronize()
161 start_time = time.time()
@@ -168,6 +171,8 @@ def test_contexted_kv_attention(
171 b_seq_len,
172 b_ctx_len,
173 max_input_len,
174 + k_scale,
175 + v_scale,
176 sliding_window=sliding_window)
177 torch.cuda.synchronize()
178 end_time = time.time()
@@ -366,6 +371,7 @@ def test_contexted_kv_attention_alibi(
371 # to V_cache[num_blocks, num_kv_heads, head_size, block_size]
372 v_cache = v_cache.view(-1, block_size, num_kv_heads,
373 head_size).permute(0, 2, 3, 1).contiguous()
374 + k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
375
376 # Warm up the Triton kernel by calling it once before actually measuring
377 # generation time
@@ -381,6 +387,8 @@ def test_contexted_kv_attention_alibi(
387 b_seq_len,
388 b_ctx_len,
389 max_input_len,
390 + k_scale,
391 + v_scale,
392 alibi_slopes=alibi_slopes)
393 torch.cuda.synchronize()
394 start_time = time.time()
@@ -396,6 +404,8 @@ def test_contexted_kv_attention_alibi(
404 b_seq_len,
405 b_ctx_len,
406 max_input_len,
407 + k_scale,
408 + v_scale,
409 alibi_slopes=alibi_slopes)
410 torch.cuda.synchronize()
411 end_time = time.time()
tests/kernels/utils.py
+2
@@ -909,6 +909,7 @@ def make_test_metadata(
909 num_prefills=num_prefills,
910 slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
911 multi_modal_placeholder_index_maps=None,
912 + enable_kv_scales_calculation=True,
913 num_prefill_tokens=num_prefill_tokens,
914 num_decode_tokens=num_decode_tokens,
915 seq_lens=seq_lens,
@@ -958,6 +959,7 @@ def make_test_metadata(
959 num_prefills=num_prefills,
960 slot_mapping=kv_mmap.slot_mapping,
961 multi_modal_placeholder_index_maps=None,
962 + enable_kv_scales_calculation=True,
963 num_prefill_tokens=num_prefill_tokens,
964 num_decode_tokens=num_decode_tokens,
965 seq_lens=seq_lens,
tests/models/decoder_only/language/test_fp8.py
+4 -11
@@ -19,18 +19,17 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
19 @pytest.mark.skipif(not is_quant_method_supported("fp8"),
20 reason="fp8 is not supported on this GPU type.")
21 @pytest.mark.parametrize(
22 - "kv_cache_dtype,base_model,test_model,scale_path",
22 + "kv_cache_dtype,base_model,test_model",
23 [
24 # Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
25 ("fp8_e4m3", "meta-llama/Llama-3.2-1B-Instruct",
26 - "nm-testing/Llama-3.2-1B-Instruct-FP8-KV", None),
26 + "nm-testing/Llama-3.2-1B-Instruct-FP8-KV"),
27 # Test FP16 checkpoint w. fp8_e5m2 kv-cache.
28 ("fp8_e5m2", "meta-llama/Llama-3.2-1B-Instruct",
29 - "meta-llama/Llama-3.2-1B-Instruct", None),
29 + "meta-llama/Llama-3.2-1B-Instruct"),
30 # Test FP16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
31 ("fp8_e4m3", "meta-llama/Llama-2-7b-chat-hf",
32 - "meta-llama/Llama-2-7b-chat-hf",
33 - "./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json")
32 + "meta-llama/Llama-2-7b-chat-hf")
33 ])
34 # Due to low-precision numerical divergence, we only test logprob of 4 tokens
35 @pytest.mark.parametrize("max_tokens", [4])
@@ -48,7 +47,6 @@ def test_models(
47 kv_cache_dtype: str,
48 base_model: str,
49 test_model: str,
51 - scale_path: Optional[str],
50 max_tokens: int,
51 enforce_eager: bool,
52 backend: str,
@@ -76,10 +74,6 @@ def test_models(
74 baseline_outputs = vllm_model.generate_greedy_logprobs(
75 example_prompts, max_tokens, NUM_LOG_PROBS)
76
79 - extra_kwargs = {}
80 - if scale_path is not None:
81 - extra_kwargs["quantization_param_path"] = scale_path
82 -
77 with vllm_runner(
78 test_model,
79 max_model_len=MAX_MODEL_LEN,
@@ -87,7 +81,6 @@ def test_models(
81 enforce_eager=enforce_eager,
82 kv_cache_dtype=kv_cache_dtype,
83 disable_async_output_proc=disable_async_output_proc,
90 - **extra_kwargs,
84 ) as vllm_model:
85 test_outputs = vllm_model.generate_greedy_logprobs(
86 example_prompts, max_tokens, NUM_LOG_PROBS)
tests/worker/test_model_input.py
+3
@@ -74,6 +74,7 @@ def test_model_runner_input():
74 num_decode_tokens=3,
75 slot_mapping=torch.zeros(1),
76 multi_modal_placeholder_index_maps=None,
77 + enable_kv_scales_calculation=True,
78 )
79 model_input = ModelInputForGPUWithSamplingMetadata(
80 input_tokens=torch.ones(10),
@@ -126,6 +127,7 @@ def test_embedding_model_runner_input():
127 num_decode_tokens=3,
128 slot_mapping=torch.zeros(1),
129 multi_modal_placeholder_index_maps=None,
130 + enable_kv_scales_calculation=True,
131 )
132 model_input = ModelInputForGPUWithPoolingMetadata(
133 input_tokens=torch.ones(10),
@@ -177,6 +179,7 @@ def test_multi_step_model_runner_input():
179 num_decode_tokens=3,
180 slot_mapping=torch.zeros(1),
181 multi_modal_placeholder_index_maps=None,
182 + enable_kv_scales_calculation=True,
183 )
184 frozen_model_input = ModelInputForGPUWithSamplingMetadata(
185 input_tokens=torch.ones(10),
vllm/_custom_ops.py
+10 -10
@@ -48,8 +48,8 @@ def paged_attention_v1(
48 max_seq_len: int,
49 alibi_slopes: Optional[torch.Tensor],
50 kv_cache_dtype: str,
51 - k_scale: float,
52 - v_scale: float,
51 + k_scale: torch.Tensor,
52 + v_scale: torch.Tensor,
53 tp_rank: int = 0,
54 blocksparse_local_blocks: int = 0,
55 blocksparse_vert_stride: int = 0,
@@ -80,8 +80,8 @@ def paged_attention_v2(
80 max_seq_len: int,
81 alibi_slopes: Optional[torch.Tensor],
82 kv_cache_dtype: str,
83 - k_scale: float,
84 - v_scale: float,
83 + k_scale: torch.Tensor,
84 + v_scale: torch.Tensor,
85 tp_rank: int = 0,
86 blocksparse_local_blocks: int = 0,
87 blocksparse_vert_stride: int = 0,
@@ -112,8 +112,8 @@ def paged_attention_rocm(
112 max_seq_len: int,
113 alibi_slopes: Optional[torch.Tensor],
114 kv_cache_dtype: str,
115 - k_scale: float,
116 - v_scale: float,
115 + k_scale: torch.Tensor,
116 + v_scale: torch.Tensor,
117 ) -> None:
118 torch.ops._rocm_C.paged_attention(out, exp_sum, max_logits, tmp_out, query,
119 key_cache, value_cache, num_kv_heads,
@@ -956,8 +956,8 @@ def reshape_and_cache(
956 value_cache: torch.Tensor,
957 slot_mapping: torch.Tensor,
958 kv_cache_dtype: str,
959 - k_scale: float,
960 - v_scale: float,
959 + k_scale: torch.Tensor,
960 + v_scale: torch.Tensor,
961 ) -> None:
962 torch.ops._C_cache_ops.reshape_and_cache(key, value, key_cache,
963 value_cache, slot_mapping,
@@ -971,8 +971,8 @@ def reshape_and_cache_flash(
971 value_cache: torch.Tensor,
972 slot_mapping: torch.Tensor,
973 kv_cache_dtype: str,
974 - k_scale: float,
975 - v_scale: float,
974 + k_scale: torch.Tensor,
975 + v_scale: torch.Tensor,
976 ) -> None:
977 torch.ops._C_cache_ops.reshape_and_cache_flash(key, value, key_cache,
978 value_cache, slot_mapping,
vllm/attention/backends/abstract.py
+8 -2
@@ -123,6 +123,10 @@ class AttentionMetadata:
123 multi_modal_placeholder_index_maps: Optional[Dict[
124 str, MultiModalPlaceholderMap.IndexMap]]
125
126 + # Enable/disable KV scales calculation. This is so that we can disable the
127 + # calculation until after prefill and cuda graph capture.
128 + enable_kv_scales_calculation: bool
129 +
130 @property
131 @abstractmethod
132 def prefill_metadata(self) -> Optional["AttentionMetadata"]:
@@ -226,8 +230,10 @@ class AttentionMetadataBuilder(ABC, Generic[T]):
230
231 class AttentionLayer(Protocol):
232
229 - _k_scale: float
230 - _v_scale: float
233 + _k_scale: torch.Tensor
234 + _v_scale: torch.Tensor
235 + _k_scale_float: float
236 + _v_scale_float: float
237
238 def forward(
239 self,
vllm/attention/backends/blocksparse_attn.py
+2
@@ -222,6 +222,7 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
222 slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
223 multi_modal_placeholder_index_maps=self.
224 multi_modal_placeholder_index_maps,
225 + enable_kv_scales_calculation=self.enable_kv_scales_calculation,
226 seq_lens=self.seq_lens[:self.num_prefills],
227 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
228 max_query_len=self.max_query_len,
@@ -251,6 +252,7 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
252 num_decode_tokens=self.num_decode_tokens,
253 slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
254 multi_modal_placeholder_index_maps=None,
255 + enable_kv_scales_calculation=False,
256 seq_lens=None,
257 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
258 max_query_len=None,
vllm/attention/backends/flash_attn.py
+4 -1
@@ -230,6 +230,7 @@ class FlashAttentionMetadata(AttentionMetadata):
230 slot_mapping=slot_mapping,
231 multi_modal_placeholder_index_maps=self.
232 multi_modal_placeholder_index_maps,
233 + enable_kv_scales_calculation=self.enable_kv_scales_calculation,
234 seq_lens=seq_lens,
235 seq_lens_tensor=seq_lens_tensor,
236 max_query_len=self.max_query_len,
@@ -274,6 +275,7 @@ class FlashAttentionMetadata(AttentionMetadata):
275 num_decode_tokens=self.num_decode_tokens,
276 slot_mapping=slot_mapping,
277 multi_modal_placeholder_index_maps=None,
278 + enable_kv_scales_calculation=True,
279 seq_lens=None,
280 seq_lens_tensor=seq_lens_tensor,
281 max_decode_query_len=self.max_decode_query_len,
@@ -557,6 +559,7 @@ class FlashAttentionMetadataBuilder(
559 num_decode_tokens=num_decode_tokens,
560 seq_lens=seq_lens,
561 multi_modal_placeholder_index_maps=placeholder_index_maps,
562 + enable_kv_scales_calculation=True,
563 seq_lens_tensor=seq_lens_tensor,
564 max_query_len=max_query_len,
565 max_decode_query_len=max_decode_query_len,
@@ -675,7 +678,7 @@ class FlashAttentionImpl(AttentionImpl):
678 NOTE: It in-place updates the output tensor.
679 """
680 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
678 - assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
681 + assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0, (
682 "key/v_scale is not supported in FlashAttention.")
683
684 assert output is not None, "Output tensor must be provided."
vllm/attention/backends/flashinfer.py
+6 -4
@@ -219,6 +219,7 @@ class FlashInferState(AttentionState):
219 num_prefills=0,
220 slot_mapping=self._graph_slot_mapping[:batch_size],
221 multi_modal_placeholder_index_maps=None,
222 + enable_kv_scales_calculation=False,
223 num_prefill_tokens=0,
224 num_decode_tokens=batch_size,
225 max_prefill_seq_len=0,
@@ -733,6 +734,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
734 num_prefills=self.num_prefills,
735 slot_mapping=slot_mapping_tensor,
736 multi_modal_placeholder_index_maps=placeholder_index_maps,
737 + enable_kv_scales_calculation=False,
738 num_prefill_tokens=self.num_prefill_tokens,
739 num_decode_tokens=num_decode_tokens,
740 max_prefill_seq_len=max_prefill_seq_len,
@@ -888,8 +890,8 @@ class FlashInferImpl(AttentionImpl):
890 kv_cache,
891 logits_soft_cap=logits_soft_cap,
892 causal=True,
891 - k_scale=layer._k_scale,
892 - v_scale=layer._v_scale,
893 + k_scale=layer._k_scale_float,
894 + v_scale=layer._v_scale_float,
895 window_left=window_left)
896 if decode_meta := attn_metadata.decode_metadata:
897 assert decode_meta is not None
@@ -899,8 +901,8 @@ class FlashInferImpl(AttentionImpl):
901 kv_cache,
902 sm_scale=softmax_scale,
903 logits_soft_cap=logits_soft_cap,
902 - k_scale=layer._k_scale,
903 - v_scale=layer._v_scale,
904 + k_scale=layer._k_scale_float,
905 + v_scale=layer._v_scale_float,
906 window_left=window_left)
907
908 if prefill_output is None and decode_output is not None:
vllm/attention/backends/ipex_attn.py
+1 -1
@@ -193,7 +193,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
193 Returns:
194 shape = [num_tokens, num_heads * head_size]
195 """
196 - assert layer._k_scale == 1.0 and layer._v_scale == 1.0
196 + assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0
197 num_tokens, hidden_size = query.shape
198 # Reshape the query, key, and value tensors.
199 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/pallas.py
+1 -1
@@ -173,7 +173,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
173 Returns:
174 shape = [batch_size, seq_len, num_heads * head_size]
175 """
176 - assert layer._k_scale == 1.0 and layer._v_scale == 1.0
176 + assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0
177 batch_size, seq_len, hidden_size = query.shape
178 query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
179 key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/placeholder_attn.py
+3
@@ -140,6 +140,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
140 slot_mapping=slot_mapping,
141 multi_modal_placeholder_index_maps=self.
142 multi_modal_placeholder_index_maps,
143 + enable_kv_scales_calculation=self.enable_kv_scales_calculation,
144 seq_lens=self.seq_lens[:self.num_prefills],
145 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
146 max_decode_query_len=0,
@@ -173,6 +174,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
174 num_decode_tokens=self.num_decode_tokens,
175 slot_mapping=slot_mapping,
176 multi_modal_placeholder_index_maps=None,
177 + enable_kv_scales_calculation=True,
178 seq_lens=None,
179 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
180 max_decode_query_len=self.max_decode_query_len,
@@ -380,6 +382,7 @@ class PlaceholderAttentionMetadataBuilder(
382 num_prefills=self.num_prefills,
383 slot_mapping=slot_mapping,
384 multi_modal_placeholder_index_maps=placeholder_index_maps,
385 + enable_kv_scales_calculation=True,
386 num_prefill_tokens=self.num_prefill_tokens,
387 num_decode_tokens=num_decode_tokens,
388 seq_lens=seq_lens,
vllm/attention/backends/rocm_flash_attn.py
+2
@@ -153,6 +153,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
153 slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
154 multi_modal_placeholder_index_maps=self.
155 multi_modal_placeholder_index_maps,
156 + enable_kv_scales_calculation=self.enable_kv_scales_calculation,
157 seq_lens=self.seq_lens[:self.num_prefills],
158 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
159 max_query_len=self.max_query_len,
@@ -182,6 +183,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
183 num_decode_tokens=self.num_decode_tokens,
184 slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
185 multi_modal_placeholder_index_maps=None,
186 + enable_kv_scales_calculation=True,
187 seq_lens=None,
188 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
189 max_query_len=None,
vllm/attention/backends/torch_sdpa.py
+1 -1
@@ -379,6 +379,7 @@ class TorchSDPAMetadataBuilder(AttentionMetadataBuilder[TorchSDPAMetadata]):
379 prefill_block_tables=prefill_block_tables,
380 slot_mapping=slot_mapping,
381 multi_modal_placeholder_index_maps=placeholder_index_maps,
382 + enable_kv_scales_calculation=False,
383 )
384
385 return attn_metadata
@@ -454,7 +455,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
455 Returns:
456 shape = [num_tokens, num_heads * head_size]
457 """
457 - assert layer._k_scale == 1.0 and layer._v_scale == 1.0
458 attn_type = self.attn_type
459 if (attn_type == AttentionType.ENCODER
460 and (not attn_metadata.is_all_encoder_attn_metadata_set)):
vllm/attention/backends/utils.py
+2
@@ -265,6 +265,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
265 num_prefills=self.num_prefills,
266 slot_mapping=slot_mapping_tensor,
267 multi_modal_placeholder_index_maps=placeholder_index_maps,
268 + enable_kv_scales_calculation=True,
269 num_prefill_tokens=self.num_prefill_tokens,
270 num_decode_tokens=num_decode_tokens,
271 seq_lens=seq_lens,
@@ -317,6 +318,7 @@ class CommonAttentionState(AttentionState):
318 num_decode_tokens=batch_size,
319 slot_mapping=self._graph_slot_mapping[:batch_size],
320 multi_modal_placeholder_index_maps=None,
321 + enable_kv_scales_calculation=True,
322 seq_lens=None,
323 seq_lens_tensor=self._graph_seq_lens[:batch_size],
324 max_query_len=1,
vllm/attention/backends/xformers.py
+2
@@ -218,6 +218,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
218 slot_mapping=slot_mapping,
219 multi_modal_placeholder_index_maps=self.
220 multi_modal_placeholder_index_maps,
221 + enable_kv_scales_calculation=self.enable_kv_scales_calculation,
222 seq_lens=seq_lens,
223 seq_lens_tensor=seq_lens_tensor,
224 max_query_len=self.max_query_len,
@@ -262,6 +263,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
263 num_decode_tokens=self.num_decode_tokens,
264 slot_mapping=slot_mapping,
265 multi_modal_placeholder_index_maps=None,
266 + enable_kv_scales_calculation=True,
267 seq_lens_tensor=seq_lens_tensor,
268 max_prefill_seq_len=0,
269 max_decode_seq_len=self.max_decode_seq_len,
vllm/attention/layer.py
+26 -2
@@ -5,6 +5,7 @@ import torch
5 import torch.nn as nn
6 import torch.nn.functional as F
7
8 +import vllm.envs as envs
9 from vllm.attention import AttentionMetadata, AttentionType
10 from vllm.attention.selector import backend_name_to_enum, get_attn_backend
11 from vllm.config import CacheConfig, get_current_vllm_config
@@ -57,10 +58,12 @@ class Attention(nn.Module):
58 kv_cache_dtype = cache_config.cache_dtype
59 block_size = cache_config.block_size
60 is_attention_free = cache_config.is_attention_free
61 + calculate_kv_scales = cache_config.calculate_kv_scales
62 else:
63 kv_cache_dtype = "auto"
64 block_size = 16
65 is_attention_free = False
66 + calculate_kv_scales = False
67 if num_kv_heads is None:
68 num_kv_heads = num_heads
69
@@ -70,8 +73,15 @@ class Attention(nn.Module):
73 # expect the pre-quantized k/v_scale to be loaded along
74 # with the model weights.
75 self.kv_cache_dtype = kv_cache_dtype
73 - self._k_scale = 1.0
74 - self._v_scale = 1.0
76 + self.calculate_kv_scales = calculate_kv_scales
77 + self._k_scale = torch.tensor(1.0, dtype=torch.float32)
78 + self._v_scale = torch.tensor(1.0, dtype=torch.float32)
79 +
80 + # We also keep the float32 versions of k/v_scale for attention
81 + # backends that don't support tensors (Flashinfer)
82 + self._k_scale_float = 1.0
83 + self._v_scale_float = 1.0
84 +
85 quant_method = quant_config.get_quant_method(
86 self, prefix=prefix) if quant_config else None
87 if quant_method is not None:
@@ -127,6 +137,9 @@ class Attention(nn.Module):
137 ).parallel_config.pipeline_parallel_size)
138 ]
139
140 + self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
141 + self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
142 +
143 def forward(
144 self,
145 query: torch.Tensor,
@@ -135,6 +148,9 @@ class Attention(nn.Module):
148 kv_cache: torch.Tensor,
149 attn_metadata: AttentionMetadata,
150 ) -> torch.Tensor:
151 + if self.calculate_kv_scales and \
152 + attn_metadata.enable_kv_scales_calculation:
153 + self.calc_kv_scales(key, value)
154 if self.use_output:
155 output = torch.empty_like(query)
156 hidden_size = query.size(-1)
@@ -161,6 +177,14 @@ class Attention(nn.Module):
177 return torch.ops.vllm.unified_attention(
178 query, key, value, self.layer_name)
179
180 + def calc_kv_scales(self, key, value):
181 + self._k_scale.copy_(torch.abs(key).max() / self.k_range)
182 + self._v_scale.copy_(torch.abs(value).max() / self.v_range)
183 + self._k_scale_float = self._k_scale.item()
184 + self._v_scale_float = self._v_scale.item()
185 + # We only calculate the scales once
186 + self.calculate_kv_scales = False
187 +
188 def extra_repr(self) -> str:
189 s = f"head_size={self.impl.head_size}" # type: ignore
190 s += f", num_heads={self.impl.num_heads}" # type: ignore
vllm/attention/ops/ipex_attn.py
+8 -8
@@ -52,8 +52,8 @@ class _PagedAttention:
52 value_cache: torch.Tensor,
53 slot_mapping: torch.Tensor,
54 kv_cache_dtype: str,
55 - k_scale: float,
56 - v_scale: float,
55 + k_scale: torch.Tensor,
56 + v_scale: torch.Tensor,
57 *args,
58 ) -> None:
59 ops.reshape_and_cache(
@@ -80,8 +80,8 @@ class _PagedAttention:
80 num_kv_heads: int,
81 scale: float,
82 alibi_slopes: Optional[torch.Tensor],
83 - k_scale: float,
84 - v_scale: float,
83 + k_scale: torch.Tensor,
84 + v_scale: torch.Tensor,
85 *args,
86 ) -> None:
87 tp_rank: int = 0
@@ -149,8 +149,8 @@ class _IPEXPagedAttention(_PagedAttention):
149 value_cache: torch.Tensor,
150 slot_mapping: torch.Tensor,
151 kv_cache_dtype: str,
152 - k_scale: float,
153 - v_scale: float,
152 + k_scale: torch.Tensor,
153 + v_scale: torch.Tensor,
154 *args,
155 ) -> None:
156 ipex_modules.PagedAttention.reshape_and_cache(
@@ -170,8 +170,8 @@ class _IPEXPagedAttention(_PagedAttention):
170 num_kv_heads: int,
171 scale: float,
172 alibi_slopes: Optional[torch.Tensor],
173 - k_scale: float,
174 - v_scale: float,
173 + k_scale: torch.Tensor,
174 + v_scale: torch.Tensor,
175 *args,
176 ) -> None:
177 block_size = value_cache.shape[2]
vllm/attention/ops/paged_attn.py
+6 -6
@@ -69,8 +69,8 @@ class PagedAttention:
69 value_cache: torch.Tensor,
70 slot_mapping: torch.Tensor,
71 kv_cache_dtype: str,
72 - k_scale: float,
73 - v_scale: float,
72 + k_scale: torch.Tensor,
73 + v_scale: torch.Tensor,
74 ) -> None:
75 ops.reshape_and_cache(
76 key,
@@ -95,8 +95,8 @@ class PagedAttention:
95 num_kv_heads: int,
96 scale: float,
97 alibi_slopes: Optional[torch.Tensor],
98 - k_scale: float,
99 - v_scale: float,
98 + k_scale: torch.Tensor,
99 + v_scale: torch.Tensor,
100 tp_rank: int = 0,
101 blocksparse_local_blocks: int = 0,
102 blocksparse_vert_stride: int = 0,
@@ -204,8 +204,8 @@ class PagedAttention:
204 max_query_len: int,
205 alibi_slopes: Optional[torch.Tensor],
206 sliding_window: Optional[int],
207 - k_scale: float,
208 - v_scale: float,
207 + k_scale: torch.Tensor,
208 + v_scale: torch.Tensor,
209 ) -> torch.Tensor:
210 output = torch.empty_like(query)
211 context_attention_fwd(
vllm/attention/ops/prefix_prefill.py
+6 -6
@@ -133,7 +133,7 @@ if triton.__version__ >= "2.1.0":
133 other=0.0) # [D,N]
134
135 if k_load.dtype.is_fp8():
136 - k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
136 + k = (k_load.to(tl.float32) * tl.load(k_scale)).to(q.dtype)
137 else:
138 k = k_load
139
@@ -181,7 +181,7 @@ if triton.__version__ >= "2.1.0":
181 ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
182 other=0.0) # [N,D]
183 if v_load.dtype.is_fp8():
184 - v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
184 + v = (v_load.to(tl.float32) * tl.load(v_scale)).to(q.dtype)
185 else:
186 v = v_load
187 p = p.to(v.dtype)
@@ -564,7 +564,7 @@ if triton.__version__ >= "2.1.0":
564 other=0.0) # [D,N]
565
566 if k_load.dtype.is_fp8():
567 - k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
567 + k = (k_load.to(tl.float32) * tl.load(k_scale)).to(q.dtype)
568 else:
569 k = k_load
570
@@ -604,7 +604,7 @@ if triton.__version__ >= "2.1.0":
604 ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
605 other=0.0)
606 if v_load.dtype.is_fp8():
607 - v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
607 + v = (v_load.to(tl.float32) * tl.load(v_scale)).to(q.dtype)
608 else:
609 v = v_load
610 p = p.to(v.dtype)
@@ -713,8 +713,8 @@ if triton.__version__ >= "2.1.0":
713 b_seq_len,
714 b_ctx_len,
715 max_input_len,
716 - k_scale: float = 1.0,
717 - v_scale: float = 1.0,
716 + k_scale: torch.Tensor,
717 + v_scale: torch.Tensor,
718 alibi_slopes=None,
719 sliding_window=None):
720
vllm/config.py
+6 -10
@@ -120,11 +120,6 @@ class ModelConfig:
120 decoding draft models.
121 quantization: Quantization method that was used to quantize the model
122 weights. If None, we assume the model weights are not quantized.
123 - quantization_param_path: Path to JSON file containing scaling factors.
124 - Used to load KV cache scaling factors into the model when KV cache
125 - type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also
126 - be used to load activation and weight scaling factors when the
127 - model dtype is FP8_E4M3 on ROCm.
123 enforce_eager: Whether to enforce eager execution. If True, we will
124 disable CUDA graph and always execute the model in eager mode.
125 If False, we will use CUDA graph and eager execution in hybrid.
@@ -187,7 +182,6 @@ class ModelConfig:
182 factors.append(self.model)
183 factors.append(self.dtype)
184 factors.append(self.quantization)
190 - factors.append(self.quantization_param_path)
185 factors.append(self.revision)
186 factors.append(self.code_revision)
187 factors.append(self.trust_remote_code)
@@ -213,7 +207,6 @@ class ModelConfig:
207 max_model_len: Optional[int] = None,
208 spec_target_max_model_len: Optional[int] = None,
209 quantization: Optional[str] = None,
216 - quantization_param_path: Optional[str] = None,
210 enforce_eager: Optional[bool] = None,
211 max_seq_len_to_capture: Optional[int] = None,
212 max_logprobs: int = 20,
@@ -274,7 +267,6 @@ class ModelConfig:
267 else:
268 self.tokenizer_revision = tokenizer_revision
269 self.quantization = quantization
277 - self.quantization_param_path = quantization_param_path
270 self.enforce_eager = enforce_eager
271 self.max_seq_len_to_capture = max_seq_len_to_capture
272 self.max_logprobs = max_logprobs
@@ -1002,6 +994,7 @@ class CacheConfig:
994 sliding_window: Optional[int] = None,
995 enable_prefix_caching: bool = False,
996 cpu_offload_gb: float = 0,
997 + calculate_kv_scales: Optional[bool] = None,
998 ) -> None:
999 self.block_size = block_size
1000 self.gpu_memory_utilization = gpu_memory_utilization
@@ -1012,7 +1005,7 @@ class CacheConfig:
1005 self.sliding_window = sliding_window
1006 self.enable_prefix_caching = enable_prefix_caching
1007 self.cpu_offload_gb = cpu_offload_gb
1015 -
1008 + self.calculate_kv_scales = calculate_kv_scales
1009 self._verify_args()
1010 self._verify_cache_dtype()
1011 self._verify_prefix_caching()
@@ -1021,6 +1014,10 @@ class CacheConfig:
1014 self.num_gpu_blocks: Optional[int] = None
1015 self.num_cpu_blocks: Optional[int] = None
1016
1017 + # Set calculate_kv_scales to False if the value is unset.
1018 + if self.calculate_kv_scales is None:
1019 + self.calculate_kv_scales = False
1020 +
1021 def metrics_info(self):
1022 # convert cache_config to dict(key: str, value: str) for prometheus
1023 # metrics info
@@ -3297,7 +3294,6 @@ class VllmConfig:
3294 f"quantization={self.model_config.quantization}, "
3295 f"enforce_eager={self.model_config.enforce_eager}, "
3296 f"kv_cache_dtype={self.cache_config.cache_dtype}, "
3300 - f"quantization_param_path={self.model_config.quantization_param_path},"
3297 f" device_config={self.device_config.device}, "
3298 f"decoding_config={self.decoding_config!r}, "
3299 f"observability_config={self.observability_config!r}, "
vllm/engine/arg_utils.py
+12 -13
@@ -98,7 +98,6 @@ class EngineArgs:
98 config_format: ConfigFormat = ConfigFormat.AUTO
99 dtype: str = 'auto'
100 kv_cache_dtype: str = 'auto'
101 - quantization_param_path: Optional[str] = None
101 seed: int = 0
102 max_model_len: Optional[int] = None
103 worker_use_ray: bool = False
@@ -199,6 +198,8 @@ class EngineArgs:
198 generation_config: Optional[str] = None
199 enable_sleep_mode: bool = False
200
201 + calculate_kv_scales: Optional[bool] = None
202 +
203 def __post_init__(self):
204 if not self.tokenizer:
205 self.tokenizer = self.model
@@ -350,17 +351,6 @@ class EngineArgs:
351 help='Data type for kv cache storage. If "auto", will use model '
352 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
353 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
353 - parser.add_argument(
354 - '--quantization-param-path',
355 - type=nullable_str,
356 - default=None,
357 - help='Path to the JSON file containing the KV cache '
358 - 'scaling factors. This should generally be supplied, when '
359 - 'KV cache dtype is FP8. Otherwise, KV cache scaling factors '
360 - 'default to 1.0, which may cause accuracy issues. '
361 - 'FP8_E5M2 (without scaling) is only supported on cuda version '
362 - 'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
363 - 'supported for common inference criteria.')
354 parser.add_argument('--max-model-len',
355 type=int,
356 default=EngineArgs.max_model_len,
@@ -962,6 +952,15 @@ class EngineArgs:
952 help="Enable sleep mode for the engine. "
953 "(only cuda platform is supported)")
954
955 + parser.add_argument(
956 + '--calculate-kv-scales',
957 + action='store_true',
958 + help='This enables dynamic calculation of '
959 + 'k_scale and v_scale when kv-cache-dtype is fp8. '
960 + 'If calculate-kv-scales is false, the scales will '
961 + 'be loaded from the model checkpoint if available. '
962 + 'Otherwise, the scales will default to 1.0.')
963 +
964 return parser
965
966 @classmethod
@@ -991,7 +990,6 @@ class EngineArgs:
990 tokenizer_revision=self.tokenizer_revision,
991 max_model_len=self.max_model_len,
992 quantization=self.quantization,
994 - quantization_param_path=self.quantization_param_path,
993 enforce_eager=self.enforce_eager,
994 max_seq_len_to_capture=self.max_seq_len_to_capture,
995 max_logprobs=self.max_logprobs,
@@ -1068,6 +1066,7 @@ class EngineArgs:
1066 sliding_window=model_config.get_sliding_window(),
1067 enable_prefix_caching=self.enable_prefix_caching,
1068 cpu_offload_gb=self.cpu_offload_gb,
1069 + calculate_kv_scales=self.calculate_kv_scales,
1070 )
1071 parallel_config = ParallelConfig(
1072 pipeline_parallel_size=self.pipeline_parallel_size,
vllm/envs.py
+9
@@ -73,6 +73,8 @@ if TYPE_CHECKING:
73 VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
74 VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
75 VLLM_DISABLE_COMPILE_CACHE: bool = False
76 + K_SCALE_CONSTANT: int = 200
77 + V_SCALE_CONSTANT: int = 100
78 VLLM_SERVER_DEV_MODE: bool = False
79 VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
80
@@ -474,6 +476,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
476 "VLLM_USE_V1":
477 lambda: bool(int(os.getenv("VLLM_USE_V1", "0"))),
478
479 + # Divisor for dynamic key scale factor calculation for FP8 KV Cache
480 + "K_SCALE_CONSTANT":
481 + lambda: int(os.getenv("K_SCALE_CONSTANT", "200")),
482 +
483 + # Divisor for dynamic value scale factor calculation for FP8 KV Cache
484 + "V_SCALE_CONSTANT":
485 + lambda: int(os.getenv("V_SCALE_CONSTANT", "100")),
486 # If set, enable multiprocessing in LLM for the V1 code path.
487 "VLLM_ENABLE_V1_MULTIPROCESSING":
488 lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))),
vllm/model_executor/layers/quantization/kv_cache.py
+15 -4
@@ -3,6 +3,7 @@ import torch
3 from vllm.logger import init_logger
4 from vllm.model_executor.layers.quantization.base_config import (
5 QuantizationConfig, QuantizeMethodBase)
6 +from vllm.platforms import current_platform
7
8 logger = init_logger(__name__)
9
@@ -40,11 +41,16 @@ class BaseKVCacheMethod(QuantizeMethodBase):
41 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
42 # If the kv-cache dtype is auto, we enforce the k/v_scale to be 1.0
43 # regardless whether the kv-scale is available in the checkpoint.
43 - if layer.kv_cache_dtype != "auto":
44 + # No need to process kv scales after loading if we are going to
45 + # calculate them on the fly.
46 + if layer.kv_cache_dtype != "auto" and not layer.calculate_kv_scales:
47 if layer.k_scale > 0.0 and layer.v_scale > 0.0:
48 # We prefer to use separate k_scale and v_scale if present
49 k_scale = layer.k_scale.to("cpu").tolist()
50 v_scale = layer.v_scale.to("cpu").tolist()
51 + if current_platform.is_rocm():
52 + k_scale *= 2
53 + v_scale *= 2
54 elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
55 # If no scales were loaded (both scales are invalid negative
56 # values), use the default value of 1.0
@@ -58,6 +64,9 @@ class BaseKVCacheMethod(QuantizeMethodBase):
64 scale_to_duplicate = max(layer.k_scale, layer.v_scale)
65 k_scale = scale_to_duplicate.to("cpu").tolist()
66 v_scale = scale_to_duplicate.to("cpu").tolist()
67 + if current_platform.is_rocm():
68 + k_scale *= 2
69 + v_scale *= 2
70
71 if not isinstance(k_scale, float) or not isinstance(
72 v_scale, float):
@@ -65,9 +74,11 @@ class BaseKVCacheMethod(QuantizeMethodBase):
74 "for fp8 KV cache")
75
76 # These are used in the final Attention.forward()
68 - layer._k_scale = k_scale
69 - layer._v_scale = v_scale
70 - if (layer._k_scale == 1.0 and layer._v_scale == 1.0
77 + layer._k_scale.copy_(k_scale)
78 + layer._v_scale.copy_(v_scale)
79 + layer._k_scale_float = k_scale
80 + layer._v_scale_float = v_scale
81 + if (k_scale == 1.0 and v_scale == 1.0
82 and "e5m2" not in layer.kv_cache_dtype):
83 logger.warning_once(
84 "Using KV cache scaling factor 1.0 for fp8_e4m3. This "
vllm/model_executor/model_loader/weight_utils.py
+1 -44
@@ -6,8 +6,7 @@ import json
6 import os
7 import tempfile
8 from collections import defaultdict
9 -from typing import (Any, Callable, Dict, Generator, Iterable, List, Optional,
10 - Tuple, Union)
9 +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
10
11 import filelock
12 import gguf
@@ -23,7 +22,6 @@ from vllm.distributed import get_tensor_model_parallel_rank
22 from vllm.logger import init_logger
23 from vllm.model_executor.layers.quantization import (QuantizationConfig,
24 get_quantization_config)
26 -from vllm.model_executor.layers.quantization.schema import QuantParamSchema
25 from vllm.platforms import current_platform
26 from vllm.utils import PlaceholderModule
27
@@ -496,47 +494,6 @@ def gguf_quant_weights_iterator(
494 yield name, param
495
496
499 -def kv_cache_scales_loader(
500 - filename: str, tp_rank: int, tp_size: int, num_hidden_layers: int,
501 - model_type: Optional[str]) -> Iterable[Tuple[int, float]]:
502 - """
503 - A simple utility to read in KV cache scaling factors that have been
504 - previously serialized to disk. Used by the model to populate the appropriate
505 - KV cache scaling factors. The serialization should represent a dictionary
506 - whose keys are the TP ranks and values are another dictionary mapping layers
507 - to their KV cache scaling factors.
508 - Keep this function in sync with the output of
509 - examples/other/fp8/extract_scales.py
510 - """
511 - try:
512 - with open(filename) as f:
513 - context = {
514 - "model_type": model_type,
515 - "num_hidden_layers": num_hidden_layers,
516 - "tp_rank": tp_rank,
517 - "tp_size": tp_size,
518 - }
519 - schema_dct = json.load(f)
520 - schema = QuantParamSchema.model_validate(schema_dct,
521 - context=context)
522 - layer_scales_map = schema.kv_cache.scaling_factor[tp_rank]
523 - return layer_scales_map.items()
524 -
525 - except FileNotFoundError:
526 - logger.error("File or directory '%s' not found.", filename)
527 - except json.JSONDecodeError:
528 - logger.error("Error decoding JSON in file '%s'.", filename)
529 - except Exception:
530 - logger.exception("An error occurred while reading '%s'.", filename)
531 - # This section is reached if and only if any of the excepts are hit
532 - # Return an empty iterable (list) => no KV cache scales are loaded
533 - # which ultimately defaults to 1.0 scales
534 - logger.warning(
535 - "Defaulting to KV cache scaling factors = 1.0 for all "
536 - "layers in TP rank %d as an error occurred during loading.", tp_rank)
537 - return []
538 -
539 -
497 def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
498 """convert PySafeSlice object from safetensors to torch.Tensor
499
vllm/model_executor/models/exaone.py
+2 -33
@@ -30,8 +30,7 @@ from torch import nn
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, VllmConfig
33 -from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 - get_tensor_model_parallel_world_size)
33 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -44,9 +43,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import (
47 - default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
46 + default_weight_loader, maybe_remap_kv_scale_name)
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 -from vllm.platforms import current_platform
48 from vllm.sequence import IntermediateTensors
49 from vllm.transformers_utils.configs.exaone import ExaoneConfig
50
@@ -576,32 +574,3 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
574 weight_loader(param, loaded_weight)
575 loaded_params.add(name)
576 return loaded_params
579 -
580 - # If this function is called, it should always initialize KV cache scale
581 - # factors (or else raise an exception). Thus, handled exceptions should
582 - # make sure to leave KV cache scale factors in a known good (dummy) state
583 - def load_kv_cache_scales(self, quantization_param_path: str) -> None:
584 - tp_size = get_tensor_model_parallel_world_size()
585 - tp_rank = get_tensor_model_parallel_rank()
586 - for layer_idx, scaling_factor in kv_cache_scales_loader(
587 - quantization_param_path,
588 - tp_rank,
589 - tp_size,
590 - self.config.num_hidden_layers,
591 - self.config.__class__.model_type,
592 - ):
593 - if not isinstance(self.transformer.h[layer_idx], nn.Identity):
594 - layer_self_attn = self.transformer.h[layer_idx].attn
595 -
596 - if current_platform.is_rocm():
597 - # The scaling factor convention we are assuming is
598 - # quantized_value * scaling_factor ~= true_value
599 - # which is consistent with the practice of setting
600 - # scaling_factor = tensor_amax / FPtype_max
601 - scaling_factor *= 2
602 - if hasattr(layer_self_attn.attn, "_k_scale"):
603 - layer_self_attn.attn._k_scale = scaling_factor
604 - layer_self_attn.attn._v_scale = scaling_factor
605 - else:
606 - raise RuntimeError("Self attention has no KV cache scaling "
607 - "factor attribute!")
vllm/model_executor/models/granite.py
+2 -30
@@ -29,8 +29,7 @@ from transformers import GraniteConfig
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 from vllm.config import CacheConfig, VllmConfig
32 -from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 - get_tensor_model_parallel_world_size)
32 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -44,9 +43,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import (
47 - default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
46 + default_weight_loader, maybe_remap_kv_scale_name)
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 -from vllm.platforms import current_platform
48 from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsLoRA, SupportsPP
@@ -518,29 +516,3 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
516 weight_loader(param, loaded_weight)
517 loaded_params.add(name)
518 return loaded_params
521 -
522 - # If this function is called, it should always initialize KV cache scale
523 - # factors (or else raise an exception). Thus, handled exceptions should
524 - # make sure to leave KV cache scale factors in a known good (dummy) state
525 - def load_kv_cache_scales(self, quantization_param_path: str) -> None:
526 - tp_size = get_tensor_model_parallel_world_size()
527 - tp_rank = get_tensor_model_parallel_rank()
528 - for layer_idx, scaling_factor in kv_cache_scales_loader(
529 - quantization_param_path, tp_rank, tp_size,
530 - self.config.num_hidden_layers,
531 - self.config.__class__.model_type):
532 - if not isinstance(self.model.layers[layer_idx], nn.Identity):
533 - layer_self_attn = self.model.layers[layer_idx].self_attn
534 -
535 - if current_platform.is_rocm():
536 - # The scaling factor convention we are assuming is
537 - # quantized_value * scaling_factor ~= true_value
538 - # which is consistent with the practice of setting
539 - # scaling_factor = tensor_amax / FPtype_max
540 - scaling_factor *= 2
541 - if hasattr(layer_self_attn.attn, "_k_scale"):
542 - layer_self_attn.attn._k_scale = scaling_factor
543 - layer_self_attn.attn._v_scale = scaling_factor
544 - else:
545 - raise RuntimeError("Self attention has no KV cache scaling "
546 - "factor attribute!")
vllm/model_executor/models/llama.py
+2 -33
@@ -29,8 +29,7 @@ from transformers import LlamaConfig
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.compilation.decorators import support_torch_compile
31 from vllm.config import CacheConfig, VllmConfig
32 -from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 - get_tensor_model_parallel_world_size)
32 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -43,9 +42,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import (
46 - default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
45 + default_weight_loader, maybe_remap_kv_scale_name)
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 -from vllm.platforms import current_platform
47 from vllm.sequence import IntermediateTensors
48
49 from .interfaces import SupportsLoRA, SupportsPP
@@ -440,32 +438,6 @@ class LlamaModel(nn.Module):
438 loaded_params.add(name)
439 return loaded_params
440
443 - # If this function is called, it should always initialize KV cache scale
444 - # factors (or else raise an exception). Thus, handled exceptions should
445 - # make sure to leave KV cache scale factors in a known good (dummy) state
446 - def load_kv_cache_scales(self, quantization_param_path: str) -> None:
447 - tp_size = get_tensor_model_parallel_world_size()
448 - tp_rank = get_tensor_model_parallel_rank()
449 - for layer_idx, scaling_factor in kv_cache_scales_loader(
450 - quantization_param_path, tp_rank, tp_size,
451 - self.config.num_hidden_layers,
452 - self.config.__class__.model_type):
453 - if not isinstance(self.layers[layer_idx], nn.Identity):
454 - layer_self_attn = self.layers[layer_idx].self_attn
455 -
456 - if current_platform.is_rocm():
457 - # The scaling factor convention we are assuming is
458 - # quantized_value * scaling_factor ~= true_value
459 - # which is consistent with the practice of setting
460 - # scaling_factor = tensor_amax / FPtype_max
461 - scaling_factor *= 2
462 - if hasattr(layer_self_attn.attn, "_k_scale"):
463 - layer_self_attn.attn._k_scale = scaling_factor
464 - layer_self_attn.attn._v_scale = scaling_factor
465 - else:
466 - raise RuntimeError("Self attention has no KV cache scaling "
467 - "factor attribute!")
468 -
441
442 class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
443 packed_modules_mapping = {
@@ -593,9 +565,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
565 self.maybe_remap_mistral(name, loaded_weight)
566 for name, loaded_weight in weights)
567
596 - def load_kv_cache_scales(self, quantization_param_path: str) -> None:
597 - self.model.load_kv_cache_scales(quantization_param_path)
598 -
568 # This function is used to remap the mistral format as
569 # used by Mistral and Llama <=2
570 def maybe_remap_mistral(
vllm/model_executor/models/mllama.py
+4 -3
@@ -831,6 +831,7 @@ class MllamaTextCrossAttention(nn.Module):
831 ) -> torch.Tensor:
832 # Skip writing kv-cache for the initial profiling run.
833 if len(kv_cache.shape) > 1:
834 + i = torch.ones(1, dtype=torch.float32)
835 if self.attn.backend in (_Backend.FLASH_ATTN,
836 _Backend.FLASH_ATTN_VLLM_V1):
837 cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
@@ -843,8 +844,8 @@ class MllamaTextCrossAttention(nn.Module):
844 attn_metadata.
845 cross_slot_mapping, # type: ignore[union-attr]
846 "auto",
846 - 1.0,
847 - 1.0,
847 + i,
848 + i,
849 )
850 elif self.attn.backend in (_Backend.XFORMERS, _Backend.TORCH_SDPA):
851 key_cache, value_cache = PagedAttention.split_kv_cache(
@@ -853,7 +854,7 @@ class MllamaTextCrossAttention(nn.Module):
854 cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
855 PagedAttention.write_to_paged_cache(
856 cached_k, cached_v, key_cache, value_cache,
856 - attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
857 + attn_metadata.cross_slot_mapping, "auto", i, i)
858 else:
859 raise ValueError(
860 f"Unsupported Attention backend {self.attn.backend} "
vllm/model_executor/models/solar.py
+2 -33
@@ -30,8 +30,7 @@ from transformers import PretrainedConfig
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, VllmConfig
33 -from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 - get_tensor_model_parallel_world_size)
33 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -44,9 +43,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import (
47 - default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
46 + default_weight_loader, maybe_remap_kv_scale_name)
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 -from vllm.platforms import current_platform
48 from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsLoRA, SupportsPP
@@ -535,32 +533,3 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
533 weight_loader(param, loaded_weight)
534 loaded_params.add(name)
535 return loaded_params
538 -
539 - # If this function is called, it should always initialize KV cache scale
540 - # factors (or else raise an exception). Thus, handled exceptions should
541 - # make sure to leave KV cache scale factors in a known good (dummy) state
542 - def load_kv_cache_scales(self, quantization_param_path: str) -> None:
543 - tp_size = get_tensor_model_parallel_world_size()
544 - tp_rank = get_tensor_model_parallel_rank()
545 - for layer_idx, scaling_factor in kv_cache_scales_loader(
546 - quantization_param_path,
547 - tp_rank,
548 - tp_size,
549 - self.config.num_hidden_layers,
550 - self.config.__class__.model_type,
551 - ):
552 - if not isinstance(self.model.layers[layer_idx], nn.Identity):
553 - layer_self_attn = self.model.layers[layer_idx].self_attn
554 -
555 - if current_platform.is_rocm():
556 - # The scaling factor convention we are assuming is
557 - # quantized_value * scaling_factor ~= true_value
558 - # which is consistent with the practice of setting
559 - # scaling_factor = tensor_amax / FPtype_max
560 - scaling_factor *= 2
561 - if hasattr(layer_self_attn.attn, "_k_scale"):
562 - layer_self_attn.attn._k_scale = scaling_factor
563 - layer_self_attn.attn._v_scale = scaling_factor
564 - else:
565 - raise RuntimeError("Self attention has no KV cache scaling "
566 - "factor attribute!")
vllm/v1/attention/backends/flash_attn.py
-4
@@ -166,10 +166,6 @@ class FlashAttentionImpl(AttentionImpl):
166 Returns:
167 shape = [num_tokens, num_heads * head_size]
168 """
169 - # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
170 - assert layer._k_scale == 1.0 and layer._v_scale == 1.0, (
171 - "key/v_scale is not supported in FlashAttention.")
172 -
169 assert output is not None, "Output tensor must be provided."
170
171 if attn_metadata is None:
vllm/worker/hpu_model_runner.py
+5 -2
@@ -903,7 +903,8 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
903 num_decode_tokens=0,
904 slot_mapping=slot_mapping,
905 multi_modal_placeholder_index_maps=
906 - None # FIXME(kzawora): mutli-modality will not work here
906 + None, # FIXME(kzawora): mutli-modality will not work here
907 + enable_kv_scales_calculation=False,
908 )
909 multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
910
@@ -1057,7 +1058,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
1058 num_prefill_tokens=0,
1059 num_decode_tokens=num_decode_tokens,
1060 slot_mapping=slot_mapping,
1060 - multi_modal_placeholder_index_maps=None)
1061 + multi_modal_placeholder_index_maps=None,
1062 + enable_kv_scales_calculation=False,
1063 + )
1064 return PrepareDecodeMetadata(input_tokens=input_tokens,
1065 input_positions=input_positions,
1066 attn_metadata=attn_metadata,
vllm/worker/model_runner.py
+6 -31
@@ -3,7 +3,6 @@ import gc
3 import inspect
4 import itertools
5 import time
6 -import warnings
6 import weakref
7 from contextlib import contextmanager
8 from dataclasses import dataclass
@@ -41,7 +40,6 @@ from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
40 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
41 MultiModalKwargs, MultiModalPlaceholderMap,
42 MultiModalRegistry)
44 -from vllm.platforms import current_platform
43 from vllm.prompt_adapter.layers import PromptAdapterMapping
44 from vllm.prompt_adapter.request import PromptAdapterRequest
45 from vllm.prompt_adapter.worker_manager import (
@@ -1151,34 +1149,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1149 self.prompt_adapter_manager.create_prompt_adapter_manager(
1150 self.model))
1151
1154 - if self.kv_cache_dtype == "fp8" and (current_platform.is_rocm()
1155 - or current_platform.is_cuda()):
1156 - # Currently only ROCm accepts kv-cache scaling factors
1157 - # via quantization_param_path and this will be deprecated
1158 - # in the future.
1159 - if self.model_config.quantization_param_path is not None:
1160 - if callable(getattr(self.model, "load_kv_cache_scales", None)):
1161 - warnings.warn(
1162 - "Loading kv cache scaling factor from JSON is "
1163 - "deprecated and will be removed. Please include "
1164 - "kv cache scaling factors in the model checkpoint.",
1165 - FutureWarning,
1166 - stacklevel=2)
1167 - self.model.load_kv_cache_scales(
1168 - self.model_config.quantization_param_path)
1169 - logger.info("Loaded KV cache scaling factors from %s",
1170 - self.model_config.quantization_param_path)
1171 - else:
1172 - raise RuntimeError(
1173 - "Using FP8 KV cache and scaling factors provided but "
1174 - "model %s does not support loading scaling factors.",
1175 - self.model.__class__)
1176 - else:
1177 - logger.warning(
1178 - "Using FP8 KV cache but no scaling factors "
1179 - "provided. Defaulting to scaling factors of 1.0. "
1180 - "This may lead to less accurate results!")
1181 -
1152 if self.vllm_config.compilation_config.level ==\
1153 CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1154 backend = self.vllm_config.compilation_config.init_backend(
@@ -1366,6 +1336,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1336 dtype=self.model_config.dtype,
1337 device=self.device)
1338
1339 + # Disable KV Scale Calculation for dummy data during profile run
1340 + if model_input.attn_metadata is not None:
1341 + model_input.attn_metadata.enable_kv_scales_calculation = False
1342 +
1343 self.execute_model(model_input, kv_caches, intermediate_tensors)
1344 torch.cuda.synchronize()
1345 return
@@ -1510,7 +1484,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1484 batch_size,
1485 is_encoder_decoder_model=self.model_config.
1486 is_encoder_decoder))
1513 -
1487 + # Disable KV Scale Calculation for graph capture
1488 + attn_metadata.enable_kv_scales_calculation = False
1489 if self.lora_config:
1490 lora_mapping = LoRAMapping(
1491 **dict(index_mapping=[0] * batch_size,
vllm/worker/openvino_model_runner.py
+1
@@ -282,6 +282,7 @@ class OpenVINOModelRunner(ModelRunnerBase):
282 block_indices_begins=block_indices_begins_tensor,
283 max_context_len=max_context_len_tensor,
284 multi_modal_placeholder_index_maps=placeholder_index_maps,
285 + enable_kv_scales_calculation=False,
286 )
287
288 multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
vllm/worker/tpu_model_runner.py
+5
@@ -190,6 +190,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
190 num_decode_tokens=0,
191 slot_mapping=slot_mapping,
192 multi_modal_placeholder_index_maps=None,
193 + enable_kv_scales_calculation=False,
194 block_tables=None,
195 context_lens=None,
196 effective_query_lens=None,
@@ -208,6 +209,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
209 num_decode_tokens=0,
210 slot_mapping=slot_mapping,
211 multi_modal_placeholder_index_maps=None,
212 + enable_kv_scales_calculation=False,
213 block_tables=block_tables,
214 context_lens=context_lens,
215 effective_query_lens=effective_query_lens,
@@ -239,6 +241,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
241 num_decode_tokens=batch_size * seq_len,
242 slot_mapping=slot_mapping,
243 multi_modal_placeholder_index_maps=None,
244 + enable_kv_scales_calculation=False,
245 block_tables=block_tables,
246 context_lens=context_lens,
247 )
@@ -425,6 +428,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
428 num_decode_tokens=0,
429 slot_mapping=slot_mapping,
430 multi_modal_placeholder_index_maps=None,
431 + enable_kv_scales_calculation=False,
432 block_tables=block_tables,
433 context_lens=context_lens,
434 effective_query_lens=prompt_lens,
@@ -496,6 +500,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
500 num_decode_tokens=batch_size,
501 slot_mapping=slot_mapping,
502 multi_modal_placeholder_index_maps=None,
503 + enable_kv_scales_calculation=False,
504 block_tables=block_tables,
505 context_lens=context_lens,
506 )
vllm/worker/xpu_model_runner.py
+2
@@ -261,6 +261,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
261 is_prompt=True,
262 slot_mapping=slot_mapping,
263 multi_modal_placeholder_index_maps=placeholder_index_maps,
264 + enable_kv_scales_calculation=False,
265 seq_lens=seq_lens,
266 seqlen_q=seqlen_q,
267 max_seqlen=max_seqlen,
@@ -345,6 +346,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
346 is_prompt=False,
347 slot_mapping=slot_mapping,
348 multi_modal_placeholder_index_maps=None,
349 + enable_kv_scales_calculation=False,
350 seq_lens=seq_lens,
351 seqlen_q=torch.tensor([]),
352 max_seqlen=0,