@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2ff767b513
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Enable scaled FP8 (e4m3fn) KV cache on ROCm (AMD GPU) (#3290)

Co-authored-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com> Co-authored-by: HaiShaw <hixiao@gmail.com> Co-authored-by: AdrianAbeyta <Adrian.Abeyta@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: root <root@gt-pla-u18-08.pla.dcgpu> Co-authored-by: mawong-amd <156021403+mawong-amd@users.noreply.github.com> Co-authored-by: ttbachyinsda <ttbachyinsda@outlook.com> Co-authored-by: guofangze <guofangze@kuaishou.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: jacobthebanana <50071502+jacobthebanana@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Adrian Abeyta committed Apr 3, 2024 at 16:15 UTC 2ff767b51301e07d1e0ad5887eb26e104e2b3a8a
41 files changed +2592 -142
.gitignore
+1
@@ -181,6 +181,7 @@ _build/
181 # hip files generated by PyTorch
182 *.hip
183 *_hip*
184 +hip_compat.h
185
186 # Benchmark dataset
187 *.json
CMakeLists.txt
+1 -1
@@ -19,7 +19,7 @@ set(PYTHON_SUPPORTED_VERSIONS "3.8" "3.9" "3.10" "3.11")
19 set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
20
21 # Supported AMD GPU architectures.
22 -set(HIP_SUPPORTED_ARCHS "gfx908;gfx90a;gfx942;gfx1100")
22 +set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100")
23
24 #
25 # Supported/expected torch versions for CUDA/ROCm.
benchmarks/benchmark_latency.py
+16 -2
@@ -24,6 +24,7 @@ def main(args: argparse.Namespace):
24 dtype=args.dtype,
25 enforce_eager=args.enforce_eager,
26 kv_cache_dtype=args.kv_cache_dtype,
27 + quantization_param_path=args.quantization_param_path,
28 device=args.device,
29 ray_workers_use_nsight=args.ray_workers_use_nsight,
30 enable_chunked_prefill=args.enable_chunked_prefill,
@@ -127,10 +128,23 @@ if __name__ == '__main__':
128 parser.add_argument(
129 "--kv-cache-dtype",
130 type=str,
130 - choices=['auto', 'fp8_e5m2'],
131 + choices=['auto', 'fp8'],
132 default='auto',
133 help=
133 - 'Data type for kv cache storage. If "auto", will use model data type.')
134 + 'Data type for kv cache storage. If "auto", will use model data type. '
135 + 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
136 + 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
137 + 'common inference criteria.')
138 + parser.add_argument(
139 + '--quantization-param-path',
140 + type=str,
141 + default=None,
142 + help='Path to the JSON file containing the KV cache scaling factors. '
143 + 'This should generally be supplied, when KV cache dtype is FP8. '
144 + 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
145 + 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
146 + 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
147 + 'instead supported for common inference criteria.')
148 parser.add_argument(
149 '--profile',
150 action='store_true',
benchmarks/benchmark_throughput.py
+19 -3
@@ -72,6 +72,7 @@ def run_vllm(
72 max_model_len: Optional[int],
73 enforce_eager: bool,
74 kv_cache_dtype: str,
75 + quantization_param_path: Optional[str],
76 device: str,
77 enable_prefix_caching: bool,
78 gpu_memory_utilization: float = 0.9,
@@ -89,6 +90,7 @@ def run_vllm(
90 gpu_memory_utilization=gpu_memory_utilization,
91 enforce_eager=enforce_eager,
92 kv_cache_dtype=kv_cache_dtype,
93 + quantization_param_path=quantization_param_path,
94 device=device,
95 enable_prefix_caching=enable_prefix_caching,
96 download_dir=download_dir)
@@ -217,7 +219,8 @@ def main(args: argparse.Namespace):
219 args.seed, args.n, args.use_beam_search,
220 args.trust_remote_code, args.dtype,
221 args.max_model_len, args.enforce_eager,
220 - args.kv_cache_dtype, args.device,
222 + args.kv_cache_dtype,
223 + args.quantization_param_path, args.device,
224 args.enable_prefix_caching,
225 args.gpu_memory_utilization, args.download_dir)
226 elif args.backend == "hf":
@@ -306,10 +309,23 @@ if __name__ == "__main__":
309 parser.add_argument(
310 "--kv-cache-dtype",
311 type=str,
309 - choices=["auto", "fp8_e5m2"],
312 + choices=["auto", "fp8"],
313 default="auto",
314 help=
312 - 'Data type for kv cache storage. If "auto", will use model data type.')
315 + 'Data type for kv cache storage. If "auto", will use model data type. '
316 + 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
317 + 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
318 + 'common inference criteria.')
319 + parser.add_argument(
320 + '--quantization-param-path',
321 + type=str,
322 + default=None,
323 + help='Path to the JSON file containing the KV cache scaling factors. '
324 + 'This should generally be supplied, when KV cache dtype is FP8. '
325 + 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
326 + 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
327 + 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
328 + 'instead supported for common inference criteria.')
329 parser.add_argument(
330 "--device",
331 type=str,
benchmarks/kernels/benchmark_paged_attention.py
+10 -3
@@ -97,6 +97,9 @@ def main(
97 torch.cuda.cudart().cudaProfilerStart()
98 start_time = time.perf_counter()
99
100 + # Using default kv_scale
101 + kv_scale = 1.0
102 +
103 for _ in range(num_iters):
104 if version == "v1":
105 ops.paged_attention_v1(
@@ -112,6 +115,7 @@ def main(
115 max_context_len,
116 alibi_slopes,
117 kv_cache_dtype,
118 + kv_scale,
119 )
120 elif version == "v2":
121 ops.paged_attention_v2(
@@ -130,6 +134,7 @@ def main(
134 max_context_len,
135 alibi_slopes,
136 kv_cache_dtype,
137 + kv_scale,
138 )
139 else:
140 raise ValueError(f"Invalid version: {version}")
@@ -179,11 +184,13 @@ if __name__ == '__main__':
184 parser.add_argument(
185 "--kv-cache-dtype",
186 type=str,
182 - choices=["auto", "fp8_e5m2"],
187 + choices=["auto", "fp8"],
188 default="auto",
189 help=
185 - 'Data type for kv cache storage. If "auto", will use model data type.')
186 - parser.add_argument("--device", type=str, choices=["cuda"], default="cuda")
190 + 'Data type for kv cache storage. If "auto", will use model data type. '
191 + 'FP8_E5M2 (without scaling) is only supported on cuda version greater '
192 + 'than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for '
193 + 'common inference criteria.')
194 args = parser.parse_args()
195 print(args)
196
cmake/utils.cmake
+1
@@ -117,6 +117,7 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
117
118 list(APPEND GPU_FLAGS
119 "-DUSE_ROCM"
120 + "-DENABLE_FP8_E4M3"
121 "-U__HIP_NO_HALF_CONVERSIONS__"
122 "-U__HIP_NO_HALF_OPERATORS__"
123 "-fno-gpu-rdc")
csrc/attention/attention_dtypes.h
+1 -1
@@ -4,4 +4,4 @@
4 #include "dtype_float16.cuh"
5 #include "dtype_float32.cuh"
6 #include "dtype_bfloat16.cuh"
7 -#include "dtype_fp8_e5m2.cuh"
7 +#include "dtype_fp8.cuh"
csrc/attention/attention_kernels.cu
+79 -44
@@ -22,12 +22,26 @@
22
23 #include "attention_dtypes.h"
24 #include "attention_utils.cuh"
25 -#ifdef ENABLE_FP8_E5M2
25 +
26 +#if defined(ENABLE_FP8_E5M2)
27 #include "../quantization/fp8_e5m2_kvcache/quant_utils.cuh"
28 +#elif defined(ENABLE_FP8_E4M3)
29 +#include "../quantization/fp8/amd_detail/quant_utils.cuh"
30 #endif
31
32 #include <algorithm>
33
34 +#ifdef USE_ROCM
35 + #include <hip/hip_bf16.h>
36 + typedef __hip_bfloat16 __nv_bfloat16;
37 +#endif
38 +
39 +#ifndef USE_ROCM
40 +#define WARP_SIZE 32
41 +#else
42 +#define WARP_SIZE warpSize
43 +#endif
44 +
45 #define MAX(a, b) ((a) > (b) ? (a) : (b))
46 #define MIN(a, b) ((a) < (b) ? (a) : (b))
47 #define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
@@ -78,7 +92,7 @@ template<
92 int HEAD_SIZE,
93 int BLOCK_SIZE,
94 int NUM_THREADS,
81 - bool IS_FP8_E5M2_KV_CACHE,
95 + bool IS_FP8_KV_CACHE,
96 int PARTITION_SIZE = 0> // Zero means no partitioning.
97 __device__ void paged_attention_kernel(
98 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -95,7 +109,8 @@ __device__ void paged_attention_kernel(
109 const float* __restrict__ alibi_slopes, // [num_heads]
110 const int q_stride,
111 const int kv_block_stride,
98 - const int kv_head_stride) {
112 + const int kv_head_stride,
113 + const float kv_scale) {
114 const int seq_idx = blockIdx.y;
115 const int partition_idx = blockIdx.z;
116 const int max_num_partitions = gridDim.z;
@@ -142,7 +157,7 @@ __device__ void paged_attention_kernel(
157 constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
158 using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
159 using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
145 -#ifdef ENABLE_FP8_E5M2
160 +#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
161 using Quant_vec = typename Vec<cache_t, VEC_SIZE>::Type;
162 #endif
163
@@ -208,11 +223,16 @@ __device__ void paged_attention_kernel(
223 const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
224 const int offset1 = (vec_idx * VEC_SIZE) / x;
225 const int offset2 = (vec_idx * VEC_SIZE) % x;
211 - if constexpr (IS_FP8_E5M2_KV_CACHE) {
212 -#ifdef ENABLE_FP8_E5M2
226 + if constexpr (IS_FP8_KV_CACHE) {
227 +#if defined(ENABLE_FP8_E5M2)
228 Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
229 // Vector conversion from Quant_vec to K_vec.
230 k_vecs[j] = fp8_e5m2_unscaled::vec_conversion<K_vec, Quant_vec>(k_vec_quant);
231 +#elif defined(ENABLE_FP8_E4M3)
232 + Quant_vec k_vec_quant = *reinterpret_cast<const Quant_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
233 + // Vector conversion from Quant_vec to K_vec. Use scaled_vec_conversion to convert FP8_E4M3 quantized k
234 + // cache vec to k vec in higher precision (FP16, BFloat16, etc.)
235 + k_vecs[j] = fp8_e4m3::scaled_vec_conversion<K_vec, Quant_vec>(k_vec_quant, kv_scale);
236 #else
237 assert(false);
238 #endif
@@ -292,7 +312,7 @@ __device__ void paged_attention_kernel(
312 constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
313 using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
314 using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
295 -#ifdef ENABLE_FP8_E5M2
315 +#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
316 using V_quant_vec = typename Vec<cache_t, V_VEC_SIZE>::Type;
317 #endif
318 using Float_L_vec = typename FloatVec<L_vec>::Type;
@@ -328,11 +348,16 @@ __device__ void paged_attention_kernel(
348 if (row_idx < HEAD_SIZE) {
349 const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
350 V_vec v_vec;
331 - if constexpr (IS_FP8_E5M2_KV_CACHE) {
332 -#ifdef ENABLE_FP8_E5M2
351 + if constexpr (IS_FP8_KV_CACHE) {
352 +#if defined(ENABLE_FP8_E5M2)
353 V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
354 // Vector conversion from V_quant_vec to V_vec.
355 v_vec = fp8_e5m2_unscaled::vec_conversion<V_vec, V_quant_vec>(v_quant_vec);
356 +#elif defined(ENABLE_FP8_E4M3)
357 + V_quant_vec v_quant_vec = *reinterpret_cast<const V_quant_vec*>(v_ptr + offset);
358 + // Vector conversion from V_quant_vec to V_vec. Use scaled_vec_conversion to convert
359 + // FP8_E4M3 quantized v cache vec to v vec in higher precision (FP16, BFloat16, etc.)
360 + v_vec = fp8_e4m3::scaled_vec_conversion<V_vec, V_quant_vec>(v_quant_vec, kv_scale);
361 #else
362 assert(false);
363 #endif
@@ -423,7 +448,7 @@ template<
448 int HEAD_SIZE,
449 int BLOCK_SIZE,
450 int NUM_THREADS,
426 - bool IS_FP8_E5M2_KV_CACHE>
451 + bool IS_FP8_KV_CACHE>
452 __global__ void paged_attention_v1_kernel(
453 scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
454 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
@@ -437,11 +462,12 @@ __global__ void paged_attention_v1_kernel(
462 const float* __restrict__ alibi_slopes, // [num_heads]
463 const int q_stride,
464 const int kv_block_stride,
440 - const int kv_head_stride) {
441 - paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE>(
465 + const int kv_head_stride,
466 + const float kv_scale) {
467 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE>(
468 /* exp_sums */ nullptr, /* max_logits */ nullptr,
469 out, q, k_cache, v_cache, num_kv_heads, scale, block_tables, context_lens,
444 - max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride);
470 + max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride, kv_head_stride, kv_scale);
471 }
472
473 // Grid: (num_heads, num_seqs, max_num_partitions).
@@ -451,7 +477,7 @@ template<
477 int HEAD_SIZE,
478 int BLOCK_SIZE,
479 int NUM_THREADS,
454 - bool IS_FP8_E5M2_KV_CACHE,
480 + bool IS_FP8_KV_CACHE,
481 int PARTITION_SIZE>
482 __global__ void paged_attention_v2_kernel(
483 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -468,11 +494,12 @@ __global__ void paged_attention_v2_kernel(
494 const float* __restrict__ alibi_slopes, // [num_heads]
495 const int q_stride,
496 const int kv_block_stride,
471 - const int kv_head_stride) {
472 - paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE>(
497 + const int kv_head_stride,
498 + const float kv_scale) {
499 + paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, IS_FP8_KV_CACHE, PARTITION_SIZE>(
500 exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
501 block_tables, context_lens, max_num_blocks_per_seq, alibi_slopes,
475 - q_stride, kv_block_stride, kv_head_stride);
502 + q_stride, kv_block_stride, kv_head_stride, kv_scale);
503 }
504
505 // Grid: (num_heads, num_seqs).
@@ -579,9 +606,9 @@ __global__ void paged_attention_v2_reduce_kernel(
606 #define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
607 VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
608 ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
582 - IS_FP8_E5M2_KV_CACHE>), shared_mem_size); \
609 + IS_FP8_KV_CACHE>), shared_mem_size); \
610 vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
584 - IS_FP8_E5M2_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
611 + IS_FP8_KV_CACHE><<<grid, block, shared_mem_size, stream>>>( \
612 out_ptr, \
613 query_ptr, \
614 key_cache_ptr, \
@@ -594,14 +621,15 @@ __global__ void paged_attention_v2_reduce_kernel(
621 alibi_slopes_ptr, \
622 q_stride, \
623 kv_block_stride, \
597 - kv_head_stride);
624 + kv_head_stride, \
625 + kv_scale);
626
627 // TODO(woosuk): Tune NUM_THREADS.
628 template<
629 typename T,
630 typename CACHE_T,
631 int BLOCK_SIZE,
604 - bool IS_FP8_E5M2_KV_CACHE,
632 + bool IS_FP8_KV_CACHE,
633 int NUM_THREADS = 128>
634 void paged_attention_v1_launcher(
635 torch::Tensor& out,
@@ -613,7 +641,8 @@ void paged_attention_v1_launcher(
641 torch::Tensor& block_tables,
642 torch::Tensor& context_lens,
643 int max_context_len,
616 - const c10::optional<torch::Tensor>& alibi_slopes) {
644 + const c10::optional<torch::Tensor>& alibi_slopes,
645 + float kv_scale) {
646 int num_seqs = query.size(0);
647 int num_heads = query.size(1);
648 int head_size = query.size(2);
@@ -677,8 +706,8 @@ void paged_attention_v1_launcher(
706 }
707 }
708
680 -#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
681 - paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
709 +#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
710 + paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
711 out, \
712 query, \
713 key_cache, \
@@ -688,20 +717,21 @@ void paged_attention_v1_launcher(
717 block_tables, \
718 context_lens, \
719 max_context_len, \
691 - alibi_slopes);
720 + alibi_slopes, \
721 + kv_scale);
722
723 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
724 // 1, 2, 4, 64, 128, 256.
695 -#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
725 +#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
726 switch (block_size) { \
727 case 8: \
698 - CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
728 + CALL_V1_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
729 break; \
730 case 16: \
701 - CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
731 + CALL_V1_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
732 break; \
733 case 32: \
704 - CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
734 + CALL_V1_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
735 break; \
736 default: \
737 TORCH_CHECK(false, "Unsupported block size: ", block_size); \
@@ -720,7 +750,8 @@ void paged_attention_v1(
750 int block_size,
751 int max_context_len,
752 const c10::optional<torch::Tensor>& alibi_slopes,
723 - const std::string& kv_cache_dtype) {
753 + const std::string& kv_cache_dtype,
754 + float kv_scale) {
755 if (kv_cache_dtype == "auto") {
756 if (query.dtype() == at::ScalarType::Float) {
757 CALL_V1_LAUNCHER_BLOCK_SIZE(float, float, false);
@@ -731,7 +762,7 @@ void paged_attention_v1(
762 } else {
763 TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
764 }
734 - } else if (kv_cache_dtype == "fp8_e5m2") {
765 + } else if (kv_cache_dtype == "fp8") {
766 if (query.dtype() == at::ScalarType::Float) {
767 CALL_V1_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
768 } else if (query.dtype() == at::ScalarType::Half) {
@@ -748,7 +779,7 @@ void paged_attention_v1(
779
780 #define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
781 vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, \
751 - IS_FP8_E5M2_KV_CACHE, PARTITION_SIZE> \
782 + IS_FP8_KV_CACHE, PARTITION_SIZE> \
783 <<<grid, block, shared_mem_size, stream>>>( \
784 exp_sums_ptr, \
785 max_logits_ptr, \
@@ -764,7 +795,8 @@ void paged_attention_v1(
795 alibi_slopes_ptr, \
796 q_stride, \
797 kv_block_stride, \
767 - kv_head_stride); \
798 + kv_head_stride, \
799 + kv_scale); \
800 vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, PARTITION_SIZE> \
801 <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
802 out_ptr, \
@@ -778,7 +810,7 @@ template<
810 typename T,
811 typename CACHE_T,
812 int BLOCK_SIZE,
781 - bool IS_FP8_E5M2_KV_CACHE,
813 + bool IS_FP8_KV_CACHE,
814 int NUM_THREADS = 128,
815 int PARTITION_SIZE = 512>
816 void paged_attention_v2_launcher(
@@ -794,7 +826,8 @@ void paged_attention_v2_launcher(
826 torch::Tensor& block_tables,
827 torch::Tensor& context_lens,
828 int max_context_len,
797 - const c10::optional<torch::Tensor>& alibi_slopes) {
829 + const c10::optional<torch::Tensor>& alibi_slopes,
830 + float kv_scale) {
831 int num_seqs = query.size(0);
832 int num_heads = query.size(1);
833 int head_size = query.size(2);
@@ -864,8 +897,8 @@ void paged_attention_v2_launcher(
897 }
898 }
899
867 -#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE) \
868 - paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_E5M2_KV_CACHE>( \
900 +#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
901 + paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE>( \
902 out, \
903 exp_sums, \
904 max_logits, \
@@ -878,20 +911,21 @@ void paged_attention_v2_launcher(
911 block_tables, \
912 context_lens, \
913 max_context_len, \
881 - alibi_slopes);
914 + alibi_slopes, \
915 + kv_scale);
916
917 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
918 // 1, 2, 4, 64, 128, 256.
885 -#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
919 +#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, IS_FP8_KV_CACHE) \
920 switch (block_size) { \
921 case 8: \
888 - CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_E5M2_KV_CACHE); \
922 + CALL_V2_LAUNCHER(T, CACHE_T, 8, IS_FP8_KV_CACHE); \
923 break; \
924 case 16: \
891 - CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_E5M2_KV_CACHE); \
925 + CALL_V2_LAUNCHER(T, CACHE_T, 16, IS_FP8_KV_CACHE); \
926 break; \
927 case 32: \
894 - CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_E5M2_KV_CACHE); \
928 + CALL_V2_LAUNCHER(T, CACHE_T, 32, IS_FP8_KV_CACHE); \
929 break; \
930 default: \
931 TORCH_CHECK(false, "Unsupported block size: ", block_size); \
@@ -913,7 +947,8 @@ void paged_attention_v2(
947 int block_size,
948 int max_context_len,
949 const c10::optional<torch::Tensor>& alibi_slopes,
916 - const std::string& kv_cache_dtype) {
950 + const std::string& kv_cache_dtype,
951 + float kv_scale) {
952 if (kv_cache_dtype == "auto") {
953 if (query.dtype() == at::ScalarType::Float) {
954 CALL_V2_LAUNCHER_BLOCK_SIZE(float, float, false);
@@ -924,7 +959,7 @@ void paged_attention_v2(
959 } else {
960 TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
961 }
927 - } else if (kv_cache_dtype == "fp8_e5m2") {
962 + } else if (kv_cache_dtype == "fp8") {
963 if (query.dtype() == at::ScalarType::Float) {
964 CALL_V2_LAUNCHER_BLOCK_SIZE(float, uint8_t, true);
965 } else if (query.dtype() == at::ScalarType::Half) {
csrc/attention/dtype_fp8.cuh renamed
+1 -1
@@ -8,7 +8,7 @@
8 #endif
9
10 namespace vllm {
11 -#ifdef ENABLE_FP8_E5M2
11 +#if defined(ENABLE_FP8_E5M2) || defined(ENABLE_FP8_E4M3)
12 // fp8 vector types for quantization of kv cache
13
14 template<>
csrc/cache.h
+3 -2
@@ -21,9 +21,10 @@ void reshape_and_cache(
21 torch::Tensor& key_cache,
22 torch::Tensor& value_cache,
23 torch::Tensor& slot_mapping,
24 - const std::string& kv_cache_dtype);
24 + const std::string& kv_cache_dtype,
25 + const float kv_scale);
26
27 // Just for unittest
27 -void convert_fp8_e5m2(
28 +void convert_fp8(
29 torch::Tensor& src_cache,
30 torch::Tensor& dst_cache);
csrc/cache_kernels.cu
+42 -23
@@ -4,8 +4,10 @@
4
5 #include "cuda_compat.h"
6 #include "dispatch_utils.h"
7 -#ifdef ENABLE_FP8_E5M2
7 +#if defined(ENABLE_FP8_E5M2)
8 #include "quantization/fp8_e5m2_kvcache/quant_utils.cuh"
9 +#elif defined(ENABLE_FP8_E4M3)
10 +#include "quantization/fp8/amd_detail/quant_utils.cuh"
11 #endif
12
13 #include <algorithm>
@@ -151,7 +153,7 @@ void copy_blocks(
153
154 namespace vllm {
155
154 -template<typename scalar_t, typename cache_t, bool is_fp8_e5m2_kv_cache>
156 +template<typename scalar_t, typename cache_t, bool is_fp8_kv_cache>
157 __global__ void reshape_and_cache_kernel(
158 const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
159 const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
@@ -163,7 +165,8 @@ __global__ void reshape_and_cache_kernel(
165 const int num_heads,
166 const int head_size,
167 const int block_size,
166 - const int x) {
168 + const int x,
169 + const float kv_scale) {
170 const int64_t token_idx = blockIdx.x;
171 const int64_t slot_idx = slot_mapping[token_idx];
172 if (slot_idx < 0) {
@@ -195,10 +198,13 @@ __global__ void reshape_and_cache_kernel(
198 + block_offset;
199 scalar_t tgt_key = key[src_key_idx];
200 scalar_t tgt_value = value[src_value_idx];
198 - if constexpr (is_fp8_e5m2_kv_cache) {
199 -#ifdef ENABLE_FP8_E5M2
201 + if constexpr (is_fp8_kv_cache) {
202 +#if defined(ENABLE_FP8_E5M2)
203 key_cache[tgt_key_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_key);
204 value_cache[tgt_value_idx] = fp8_e5m2_unscaled::vec_conversion<uint8_t, scalar_t>(tgt_value);
205 +#elif defined(ENABLE_FP8_E4M3)
206 + key_cache[tgt_key_idx] = fp8_e4m3::scaled_vec_conversion<uint8_t, scalar_t>(tgt_key, kv_scale);
207 + value_cache[tgt_value_idx] = fp8_e4m3::scaled_vec_conversion<uint8_t, scalar_t>(tgt_value, kv_scale);
208 #else
209 assert(false);
210 #endif
@@ -211,8 +217,8 @@ __global__ void reshape_and_cache_kernel(
217
218 } // namespace vllm
219
214 -#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, IS_FP8_E5M2_KV_CACHE) \
215 - vllm::reshape_and_cache_kernel<KV_T, CACHE_T, IS_FP8_E5M2_KV_CACHE><<<grid, block, 0, stream>>>( \
220 +#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, IS_FP8_KV_CACHE) \
221 + vllm::reshape_and_cache_kernel<KV_T, CACHE_T, IS_FP8_KV_CACHE><<<grid, block, 0, stream>>>( \
222 reinterpret_cast<KV_T*>(key.data_ptr()), \
223 reinterpret_cast<KV_T*>(value.data_ptr()), \
224 reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
@@ -223,7 +229,8 @@ __global__ void reshape_and_cache_kernel(
229 num_heads, \
230 head_size, \
231 block_size, \
226 - x);
232 + x, \
233 + kv_scale);
234
235 void reshape_and_cache(
236 torch::Tensor& key, // [num_tokens, num_heads, head_size]
@@ -231,7 +238,8 @@ void reshape_and_cache(
238 torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
239 torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
240 torch::Tensor& slot_mapping, // [num_tokens]
234 - const std::string& kv_cache_dtype)
241 + const std::string& kv_cache_dtype,
242 + const float kv_scale)
243 {
244 int num_tokens = key.size(0);
245 int num_heads = key.size(1);
@@ -254,7 +262,7 @@ void reshape_and_cache(
262 } else if (key.dtype() == at::ScalarType::BFloat16) {
263 CALL_RESHAPE_AND_CACHE(__nv_bfloat16, __nv_bfloat16, false);
264 }
257 - } else if (kv_cache_dtype == "fp8_e5m2") {
265 + } else if (kv_cache_dtype == "fp8") {
266 if (key.dtype() == at::ScalarType::Float) {
267 CALL_RESHAPE_AND_CACHE(float, uint8_t, true);
268 } else if (key.dtype() == at::ScalarType::Half) {
@@ -270,15 +278,17 @@ void reshape_and_cache(
278 namespace vllm {
279
280 template<typename Tout, typename Tin>
273 -__global__ void convert_fp8_e5m2_kernel(
281 +__global__ void convert_fp8_kernel(
282 const Tin* __restrict__ src_cache,
283 Tout* __restrict__ dst_cache,
284 const int64_t block_stride) {
285 const int64_t block_idx = blockIdx.x;
286 for (int i = threadIdx.x; i < block_stride; i += blockDim.x) {
287 int64_t idx = block_idx * block_stride + i;
280 -#ifdef ENABLE_FP8_E5M2
288 +#if defined(ENABLE_FP8_E5M2)
289 dst_cache[idx] = fp8_e5m2_unscaled::vec_conversion<Tout, Tin>(src_cache[idx]);
290 +#elif defined(ENABLE_FP8_E4M3)
291 + dst_cache[idx] = fp8_e4m3::vec_conversion<Tout, Tin>(src_cache[idx]);
292 #else
293 assert(false);
294 #endif
@@ -287,16 +297,25 @@ __global__ void convert_fp8_e5m2_kernel(
297
298 } // namespace vllm
299
290 -#define CALL_CONVERT_FP8_E5M2(Tout, Tin) \
291 - vllm::convert_fp8_e5m2_kernel<Tout, Tin><<<grid, block, 0, stream>>>( \
292 - reinterpret_cast<Tin*>(src_cache.data_ptr()), \
293 - reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
300 +#define CALL_CONVERT_FP8(Tout, Tin) \
301 + vllm::convert_fp8_kernel<Tout, Tin><<<grid, block, 0, stream>>>( \
302 + reinterpret_cast<Tin*>(src_cache.data_ptr()), \
303 + reinterpret_cast<Tout*>(dst_cache.data_ptr()), \
304 block_stride);
305
296 -void convert_fp8_e5m2(
306 +void convert_fp8(
307 torch::Tensor& src_cache,
308 torch::Tensor& dst_cache)
309 {
310 + torch::Device src_device = src_cache.device();
311 + torch::Device dst_device = dst_cache.device();
312 + TORCH_CHECK(src_device.is_cuda(), "src must be on a GPU")
313 + TORCH_CHECK(dst_device.is_cuda(), "dst must be on a GPU")
314 + TORCH_CHECK(
315 + src_device.index() == dst_device.index(),
316 + "src and dst must be on the same GPU");
317 + at::cuda::OptionalCUDAGuard device_guard(src_device);
318 +
319 int64_t num_blocks = src_cache.size(0);
320 int64_t block_stride = src_cache.stride(0);
321
@@ -305,16 +324,16 @@ void convert_fp8_e5m2(
324 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
325
326 if (src_cache.dtype() == at::ScalarType::Float) {
308 - CALL_CONVERT_FP8_E5M2(uint8_t, float);
327 + CALL_CONVERT_FP8(uint8_t, float);
328 } else if (src_cache.dtype() == at::ScalarType::Half) {
310 - CALL_CONVERT_FP8_E5M2(uint8_t, uint16_t);
329 + CALL_CONVERT_FP8(uint8_t, uint16_t);
330 } else if (src_cache.dtype() == at::ScalarType::BFloat16) {
312 - CALL_CONVERT_FP8_E5M2(uint8_t, __nv_bfloat16);
331 + CALL_CONVERT_FP8(uint8_t, __nv_bfloat16);
332 } else if (dst_cache.dtype() == at::ScalarType::Float) {
314 - CALL_CONVERT_FP8_E5M2(float, uint8_t);
333 + CALL_CONVERT_FP8(float, uint8_t);
334 } else if (dst_cache.dtype() == at::ScalarType::Half) {
316 - CALL_CONVERT_FP8_E5M2(uint16_t, uint8_t);
335 + CALL_CONVERT_FP8(uint16_t, uint8_t);
336 } else if (dst_cache.dtype() == at::ScalarType::BFloat16) {
318 - CALL_CONVERT_FP8_E5M2(__nv_bfloat16, uint8_t);
337 + CALL_CONVERT_FP8(__nv_bfloat16, uint8_t);
338 }
339 }
csrc/ops.h
+4 -2
@@ -14,7 +14,8 @@ void paged_attention_v1(
14 int block_size,
15 int max_context_len,
16 const c10::optional<torch::Tensor>& alibi_slopes,
17 - const std::string& kv_cache_dtype);
17 + const std::string& kv_cache_dtype,
18 + float kv_scale);
19
20 void paged_attention_v2(
21 torch::Tensor& out,
@@ -31,7 +32,8 @@ void paged_attention_v2(
32 int block_size,
33 int max_context_len,
34 const c10::optional<torch::Tensor>& alibi_slopes,
34 - const std::string& kv_cache_dtype);
35 + const std::string& kv_cache_dtype,
36 + float kv_scale);
37
38 void rms_norm(
39 torch::Tensor& out,
csrc/pybind.cpp
+3 -3
@@ -91,9 +91,9 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
91 &reshape_and_cache,
92 "Reshape the key and value tensors and cache them");
93 cache_ops.def(
94 - "convert_fp8_e5m2",
95 - &convert_fp8_e5m2,
96 - "Convert the key and value cache to fp8_e5m2 data type");
94 + "convert_fp8",
95 + &convert_fp8,
96 + "Convert the key and value cache to fp8 data type");
97
98 // Cuda utils
99 pybind11::module cuda_utils = m.def_submodule("cuda_utils", "vLLM cuda utils");
csrc/quantization/fp8/amd_detail/hip_float8.h new
+167
@@ -0,0 +1,167 @@
1 +#pragma once
2 +
3 +#ifdef __HIPCC__
4 +#include <hip/hip_runtime.h>
5 +#else
6 +#include <type_traits>
7 +#include <stdint.h>
8 +#include <math.h>
9 +#include <iostream>
10 +#endif
11 +
12 +#include "hip_float8_impl.h"
13 +
14 +struct alignas(1) hip_fp8
15 +{
16 + struct from_bits_t
17 + {
18 + };
19 + HIP_FP8_HOST_DEVICE static constexpr from_bits_t from_bits() { return from_bits_t(); }
20 + uint8_t data;
21 +
22 + hip_fp8() = default;
23 + HIP_FP8_HOST_DEVICE constexpr hip_fp8(const hip_fp8&) = default;
24 + HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v) = delete;
25 + explicit HIP_FP8_HOST_DEVICE constexpr hip_fp8(uint8_t v, from_bits_t)
26 + : data(v)
27 + {
28 + }
29 +
30 +#ifdef __HIP__MI300__
31 + // NOTE: ON-DEVICE... always optimal bias
32 + explicit HIP_FP8_DEVICE hip_fp8(float v)
33 + : data(hip_fp8_impl::to_fp8_from_fp32(v))
34 + {
35 + }
36 +
37 + explicit HIP_FP8_DEVICE hip_fp8(_Float16 v)
38 + : hip_fp8(static_cast<float>(v))
39 + {
40 + }
41 +
42 + // Host only implementation using s/w simulation
43 + explicit HIP_FP8_HOST
44 +#else // __HIP__MI300__
45 + // both Host and DEVICE for non-MI300 using s/w simulation
46 + explicit HIP_FP8_HOST_DEVICE
47 +#endif // __HIP__MI300__
48 + hip_fp8(float v)
49 + {
50 + data = hip_fp8_impl::to_float8<4, 3, float, true /*negative_zero_nan*/, true /*clip*/>(v);
51 + }
52 +
53 + explicit HIP_FP8_HOST_DEVICE hip_fp8(double v)
54 + : hip_fp8(static_cast<float>(v))
55 + {
56 + }
57 +
58 +#ifdef __HIP__MI300__
59 + // upcast using device specific intrinsic
60 + explicit inline HIP_FP8_DEVICE operator float() const
61 + {
62 + float fval;
63 + uint32_t i32val = static_cast<uint32_t>(data);
64 +
65 + // upcast
66 + asm volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val));
67 +
68 + return fval;
69 + }
70 +
71 + explicit inline HIP_FP8_HOST operator float() const
72 +#else // __HIP__MI300__
73 + explicit inline HIP_FP8_HOST_DEVICE operator float() const
74 +#endif // __HIP__MI300__
75 + {
76 + return hip_fp8_impl::from_float8<4, 3, float, true /*negative_zero_nan*/>(data);
77 + }
78 +};
79 +
80 +namespace std
81 +{
82 +inline hip_fp8 sin(hip_fp8 a)
83 +{
84 + return hip_fp8(sinf(float(a)));
85 +}
86 +inline hip_fp8 cos(hip_fp8 a)
87 +{
88 + return hip_fp8(cosf(float(a)));
89 +}
90 +HIP_FP8_HOST_DEVICE constexpr hip_fp8 real(const hip_fp8& a)
91 +{
92 + return a;
93 +}
94 +} // namespace std
95 +
96 +// Special operator overloading
97 +inline std::ostream& operator<<(std::ostream& os, const hip_fp8& f8)
98 +{
99 + return os << float(f8);
100 +}
101 +
102 +// all + operator overloading with mixed types
103 +// mixed types, always converts to f32, does computation in f32, and returns float
104 +inline HIP_FP8_HOST_DEVICE float operator+(const float fa, hip_fp8 b)
105 +{
106 + return (fa + float(b));
107 +}
108 +
109 +inline HIP_FP8_HOST_DEVICE float operator+(hip_fp8 a, const float fb)
110 +{
111 + return (float(a) + fb);
112 +}
113 +
114 +inline HIP_FP8_HOST_DEVICE hip_fp8 operator+(hip_fp8 a, hip_fp8 b)
115 +{
116 + return hip_fp8(float(a) + float(b));
117 +}
118 +
119 +inline HIP_FP8_HOST_DEVICE hip_fp8& operator+=(hip_fp8& a, hip_fp8 b)
120 +{
121 + return a = hip_fp8(float(a) + float(b));
122 +}
123 +
124 +// overloading multiplication, always returns float,
125 +inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, hip_fp8 b)
126 +{
127 + return float(a) * float(b);
128 +}
129 +
130 +inline HIP_FP8_HOST_DEVICE float operator*(float a, hip_fp8 b)
131 +{
132 + return (a * float(b));
133 +}
134 +
135 +inline HIP_FP8_HOST_DEVICE float operator*(hip_fp8 a, float b)
136 +{
137 + return (float(a) * b);
138 +}
139 +
140 +inline HIP_FP8_HOST_DEVICE float operator*(int32_t a, hip_fp8 b)
141 +{
142 + return ((float)a * float(b));
143 +}
144 +
145 +inline HIP_FP8_HOST_DEVICE float operator*(double a, hip_fp8 b)
146 +{
147 + return ((float)a * float(b));
148 +}
149 +
150 +// overloading for compare
151 +inline HIP_FP8_HOST_DEVICE bool operator==(hip_fp8 a, hip_fp8 b)
152 +{
153 + return (a.data == b.data);
154 +}
155 +inline HIP_FP8_HOST_DEVICE bool operator!=(hip_fp8 a, hip_fp8 b)
156 +{
157 + return (a.data != b.data);
158 +}
159 +
160 +inline HIP_FP8_HOST_DEVICE bool operator>=(hip_fp8 a, hip_fp8 b)
161 +{
162 + return static_cast<float>(a) >= static_cast<float>(b);
163 +}
164 +inline HIP_FP8_HOST_DEVICE bool operator>(hip_fp8 a, hip_fp8 b)
165 +{
166 + return static_cast<float>(a) > static_cast<float>(b);
167 +}
csrc/quantization/fp8/amd_detail/hip_float8_impl.h new
+316
@@ -0,0 +1,316 @@
1 +#pragma once
2 +
3 +#if defined(__HIPCC__) && (defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__))
4 +#define __HIP__MI300__
5 +#endif
6 +
7 +#ifdef __HIPCC__
8 +#define HIP_FP8_HOST_DEVICE __host__ __device__
9 +#define HIP_FP8_HOST __host__
10 +#define HIP_FP8_DEVICE __device__
11 +#else
12 +#define HIP_FP8_HOST_DEVICE
13 +#define HIP_FP8_HOST
14 +#define HIP_FP8_DEVICE
15 +#endif
16 +
17 +namespace hip_fp8_impl
18 +{
19 +
20 +#ifdef __HIP__MI300__
21 +HIP_FP8_DEVICE uint8_t to_fp8_from_fp32(float v)
22 +{
23 + uint8_t i8data;
24 + union {
25 + float fval;
26 + uint32_t i32val;
27 + uint8_t i8val[4]; // NOTE: not endian independent
28 + } val;
29 +
30 + uint32_t ival = 0;
31 + val.fval = v;
32 +
33 + if ((val.i32val & 0x7F800000) != 0x7F800000) { /// propagate NAN/INF, no clipping
34 + val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0);
35 + }
36 +
37 + ival = __builtin_amdgcn_cvt_pk_fp8_f32(val.fval, val.fval, ival,
38 + false); // false -> WORD0
39 + val.i32val = ival;
40 + i8data = val.i8val[0];
41 +
42 + return i8data;
43 +}
44 +#endif // __HIP__MI300__
45 +
46 +HIP_FP8_HOST inline int clz(uint32_t x)
47 +{
48 + return __builtin_clz(x);
49 +}
50 +#if defined(__HIPCC__) || defined(__CUDA_ARCH__)
51 +HIP_FP8_DEVICE inline int clz(uint32_t x)
52 +{
53 + return __clz(x);
54 +}
55 +#endif
56 +
57 +template <int we, int wm, typename T, bool negative_zero_nan, bool clip>
58 +HIP_FP8_HOST_DEVICE uint8_t to_float8(T _x, bool stoch = false, uint32_t rng = 0)
59 +{
60 +#ifdef __HIPCC__
61 + constexpr bool is_half = std::is_same<T, _Float16>::value;
62 +#else
63 + constexpr bool is_half = false;
64 +#endif
65 + constexpr bool is_float = std::is_same<T, float>::value;
66 + static_assert(wm + we == 7, "wm+we==7");
67 + static_assert(is_half || is_float, "Only half and float can be cast to f8");
68 +
69 + const int mfmt = (sizeof(T) == 4) ? 23 : 10;
70 + uint32_t x;
71 + if (sizeof(T) == 4) {
72 + x = reinterpret_cast<uint32_t&>(_x);
73 + } else {
74 + x = reinterpret_cast<uint16_t&>(_x);
75 + }
76 +
77 + uint32_t head, mantissa;
78 + int exponent, bias;
79 + uint32_t sign;
80 +
81 + if (sizeof(T) == 4) {
82 + head = x & 0xFF800000;
83 + mantissa = x & 0x7FFFFF;
84 + exponent = (head >> 23) & 0xFF;
85 + sign = head >> 31;
86 + bias = 127;
87 + } else {
88 + head = x & 0xFC00;
89 + mantissa = x & 0x3FF;
90 + exponent = (head >> 10) & 0x1F;
91 + sign = head >> 15;
92 + bias = 15;
93 + }
94 +
95 + uint32_t signed_inf = (sign << 7) + (((1 << we) - 1) << wm);
96 +
97 + // Deal with inf and NaNs
98 + if (negative_zero_nan) {
99 + if (sizeof(T) == 4) {
100 + if ((x & 0x7F800000) == 0x7F800000) {
101 + return 0x80;
102 + }
103 + } else {
104 + // if(__hisinf(x) || __hisnan(x))
105 + if ((x & 0x7C00) == 0x7C00) {
106 + return 0x80;
107 + }
108 + }
109 + } else {
110 + if (sizeof(T) == 4) {
111 + if ((x & 0x7F800000) == 0x7F800000) {
112 + return signed_inf + (mantissa != 0 ? 1 : 0);
113 + }
114 + } else {
115 + if ((x & 0x7C00) == 0x7C00) {
116 + return signed_inf + (mantissa != 0 ? 1 : 0);
117 + }
118 + }
119 + }
120 + if (x == 0) {
121 + return 0;
122 + }
123 +
124 + // First need to check if it is normal or denorm as there is a difference of
125 + // implicit 1 Then need to adjust the exponent to align with the F8 exponent,
126 + // in the meanwhile, shift The mantissa. Then for stochastic rounding, add rng
127 + // to mantissa and truncate. And for RNE, no need to add rng. Then probably
128 + // need to check whether there is carry and adjust exponent and mantissa again
129 +
130 + // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent
131 + // bits
132 + const int f8_bias = (1 << (we - 1)) - 1 + (negative_zero_nan ? 1 : 0);
133 + const int f8_denormal_act_exponent = 1 - f8_bias; // actual exponent of f8 denormal
134 + // act_exponent is the actual exponent of fp32/fp16 (after subtracting bias)
135 + // f8_exponent is the converted f8 exponent with bias encoding
136 + // exponent_diff is the diff between fp32/fp16 exponent and f8 exponent,
137 + // the difference needs to be adjusted and mantissa shifted
138 + int act_exponent, f8_exponent, exponent_diff;
139 +
140 + if (exponent == 0) { // fp32/fp16 is in denormal.
141 + /* fp32 denormal is below 2^-127 so it is usually not a concern here, we
142 +mostly concern fp16 here. In this case, f8 is usually in denormal. But there
143 +could be exceptions. fp16 denormal has exponent bias 15 while bf8 with NANOO has
144 +exponent bias 16. It means that there are some numbers in fp16 denormal but they
145 +are bf8 (NANOO) normals - smallest bf8 (NANOO) normal is 2^-15. fp16 numbers
146 +where exponent==0 (actual exponent -14) and highest bit of mantissa is 1 are bf8
147 +(NANOO) normal. In this case, the fp16 mantissa should be shift left by 1 */
148 + act_exponent = exponent - bias + 1;
149 + exponent_diff = f8_denormal_act_exponent - act_exponent; // actual exponent is exponent-bias+1 as it is denormal
150 + } else { // fp32/fp16 is normal with implicit 1
151 + act_exponent = exponent - bias;
152 + if (act_exponent <= f8_denormal_act_exponent) {
153 + /* This is the case where fp32/fp16 is normal but it is in f8 denormal
154 + range. For example fp8 nanoo mode, denormal exponent is -7, but if the
155 + fp32/fp16 actual exponent is -7, it is actually larger due to the implicit 1,
156 + Therefore it needs to be adjust to -6 and mantissa shift right by 1.
157 + So for fp32/fp16, exponent -8 is the cut point to convert to fp8 nanoo */
158 + exponent_diff = f8_denormal_act_exponent - act_exponent;
159 + } else { // both fp32/fp16 and f8 are in normal range
160 + exponent_diff = 0; // exponent_diff=0 does not mean there is no difference
161 + // for this case,
162 + // act_exponent could be larger. Just that it does not need shift mantissa
163 + }
164 + mantissa += (1 << mfmt); // Add the implicit 1 into mantissa
165 + }
166 +
167 + bool midpoint = (mantissa & ((1 << (mfmt - wm + exponent_diff)) - 1)) ==
168 + static_cast<uint32_t>(1 << (mfmt - wm + exponent_diff - 1));
169 + /* This part is a bit tricky. The judgment of whether it is a tie needs to be
170 + done before we shift right as shift right could rip off some residual part
171 + and make something not midpoint look like midpoint. For example, the fp16
172 + number 0x1002 (0 00100 0000000010), it is larger than midpoint, but after
173 + shift right by 4 bits, it would look like midpoint.
174 +*/
175 +
176 + if (exponent_diff > 0) {
177 + mantissa >>= exponent_diff;
178 + } else if (exponent_diff == -1) {
179 + mantissa <<= -exponent_diff;
180 + }
181 + bool implicit_one = mantissa & (1 << mfmt);
182 + // if there is no implicit 1, it means the f8 is denormal and need to adjust
183 + // to denorm exponent
184 + f8_exponent = (act_exponent + exponent_diff) /*actual f8 exponent*/ + f8_bias - (implicit_one ? 0 : 1);
185 +
186 + // Now we have the exponent and mantissa adjusted
187 + uint32_t drop_mask = (1 << (mfmt - wm)) - 1;
188 + bool odd = mantissa & (1 << (mfmt - wm)); // if the least significant bit that
189 + // is not truncated is 1
190 + mantissa += (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) & drop_mask;
191 +
192 + // Now we deal with overflow
193 + if (f8_exponent == 0) {
194 + if ((1 << mfmt) & mantissa) {
195 + f8_exponent = 1; // denormal overflow to become normal, promote exponent
196 + }
197 + } else {
198 + if ((1 << (mfmt + 1)) & mantissa) {
199 + mantissa >>= 1;
200 + f8_exponent++;
201 + }
202 + }
203 +
204 + mantissa >>= (mfmt - wm);
205 +
206 + // above range: quantize to maximum possible float of the same sign
207 + const int max_exp = (1 << we) - (negative_zero_nan ? 1 : 2);
208 + if (f8_exponent > max_exp) {
209 + if (clip) {
210 + mantissa = (1 << wm) - 1;
211 + f8_exponent = max_exp;
212 + } else {
213 + return signed_inf;
214 + }
215 + }
216 +
217 + if (f8_exponent == 0 && mantissa == 0) {
218 + return negative_zero_nan ? 0 : (sign << 7);
219 + }
220 + mantissa &= (1 << wm) - 1;
221 + return (sign << 7) | (f8_exponent << wm) | mantissa;
222 +}
223 +
224 +template <int we, int wm, typename T = float, bool negative_zero_nan = true>
225 +inline HIP_FP8_HOST_DEVICE T from_float8(uint8_t x)
226 +{
227 +#ifdef __HIPCC__
228 + constexpr bool is_half = std::is_same<T, _Float16>::value;
229 +#else
230 + constexpr bool is_half = false;
231 +#endif
232 + constexpr bool is_float = std::is_same<T, float>::value;
233 + static_assert(is_half || is_float, "only half and float are supported");
234 +
235 + constexpr int weo = is_half ? 5 : 8;
236 + constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7);
237 +
238 + T fInf, fNegInf, fNaN, fNeg0;
239 +
240 +#ifdef __HIPCC__
241 + if (is_half) {
242 + const uint16_t ihInf = 0x7C00;
243 + const uint16_t ihNegInf = 0xFC00;
244 + const uint16_t ihNaN = 0x7C01;
245 + const uint16_t ihNeg0 = 0x8000;
246 + fInf = reinterpret_cast<const _Float16&>(ihInf);
247 + fNegInf = reinterpret_cast<const _Float16&>(ihNegInf);
248 + fNaN = reinterpret_cast<const _Float16&>(ihNaN);
249 + fNeg0 = reinterpret_cast<const _Float16&>(ihNeg0);
250 + } else
251 +#endif
252 + if (is_float) {
253 + const uint32_t ifInf = 0x7F800000;
254 + const uint32_t ifNegInf = 0xFF800000;
255 + const uint32_t ifNaN = 0x7F800001;
256 + const uint32_t ifNeg0 = 0x80000000;
257 + fInf = reinterpret_cast<const float&>(ifInf);
258 + fNegInf = reinterpret_cast<const float&>(ifNegInf);
259 + fNaN = reinterpret_cast<const float&>(ifNaN);
260 + fNeg0 = reinterpret_cast<const float&>(ifNeg0);
261 + }
262 +
263 + if (x == 0) {
264 + return 0;
265 + }
266 +
267 + uint32_t sign = x >> 7;
268 + uint32_t mantissa = x & ((1 << wm) - 1);
269 + int exponent = (x & 0x7F) >> wm;
270 + if (negative_zero_nan) {
271 + if (x == 0x80) {
272 + return fNaN;
273 + }
274 + } else {
275 + if (x == 0x80) {
276 + return fNeg0;
277 + }
278 + if (exponent == ((1 << we) - 1)) {
279 + return (mantissa == 0) ? (sign ? fNegInf : fInf) : fNaN;
280 + }
281 + }
282 + typename std::conditional<sizeof(T) == 2, uint16_t, uint32_t>::type retval;
283 + if (we == 5 && is_half && !negative_zero_nan) {
284 + retval = x << 8;
285 + return reinterpret_cast<const T&>(retval);
286 + }
287 +
288 + const int exp_low_cutoff = (1 << (weo - 1)) - (1 << (we - 1)) + 1 - (negative_zero_nan ? 1 : 0);
289 +
290 + // subnormal input
291 + if (exponent == 0) {
292 + // guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above
293 + int sh = 1 + clz(mantissa) - (32 - wm);
294 + mantissa <<= sh;
295 + exponent += 1 - sh;
296 + mantissa &= ((1 << wm) - 1);
297 + }
298 + exponent += exp_low_cutoff - 1;
299 + mantissa <<= wmo - wm;
300 +
301 + // subnormal output (occurs when T=half, we=5, negative_zero_nan=true)
302 + if (exponent <= 0) {
303 + mantissa |= 1 << wmo;
304 + mantissa >>= 1 - exponent;
305 + exponent = 0;
306 + }
307 +
308 + if (sizeof(T) == 2) {
309 + retval = (sign << 15) | (exponent << 10) | mantissa;
310 + } else {
311 + retval = (sign << 31) | (exponent << 23) | mantissa;
312 + }
313 + return reinterpret_cast<const T&>(retval);
314 +}
315 +
316 +} // namespace hip_fp8_impl
csrc/quantization/fp8/amd_detail/quant_utils.cuh new
+517
@@ -0,0 +1,517 @@
1 +#pragma once
2 +#include "hip_float8.h"
3 +
4 +#include <hip/hip_fp16.h>
5 +#include <hip/hip_bf16.h>
6 +#include <hip/hip_bfloat16.h>
7 +
8 +#include "../../../attention/dtype_float32.cuh"
9 +#include "../../../attention/dtype_bfloat16.cuh"
10 +
11 +namespace vllm
12 +{
13 +namespace fp8_e4m3 {
14 +template <typename Tout, typename Tin>
15 +__inline__ __device__ Tout vec_conversion(const Tin& x)
16 +{
17 + return x;
18 +}
19 +
20 +template <typename Tout, typename Tin>
21 +__inline__ __device__ Tout scaled_vec_conversion(const Tin& x, const float scale)
22 +{
23 + return x;
24 +}
25 +
26 +// fp8 -> half
27 +template <>
28 +__inline__ __device__ uint16_t vec_conversion<uint16_t, uint8_t>(const uint8_t& a)
29 +{
30 + hip_fp8 f8{a, hip_fp8::from_bits()};
31 + __half_raw res;
32 + res.data = static_cast<float>(f8);
33 + return res.x;
34 +}
35 +
36 +// fp8x2 -> half2
37 +template <>
38 +__inline__ __device__ uint32_t vec_conversion<uint32_t, uint16_t>(const uint16_t& a)
39 +{
40 +#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
41 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
42 + union {
43 + __half2_raw h2r;
44 + uint32_t ui32;
45 + } tmp;
46 + tmp.h2r.x.data = f2[0];
47 + tmp.h2r.y.data = f2[1];
48 + return tmp.ui32;
49 +#else
50 + union {
51 + uint16_t u16[2];
52 + uint32_t u32;
53 + } tmp;
54 +
55 + tmp.u16[0] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a));
56 + tmp.u16[1] = vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U));
57 + return tmp.u32;
58 +#endif
59 +}
60 +
61 +// fp8x4 -> half2x2
62 +template <>
63 +__inline__ __device__ uint2 vec_conversion<uint2, uint32_t>(const uint32_t& a)
64 +{
65 + union {
66 + uint2 u32x2;
67 + uint32_t u32[2];
68 + } tmp;
69 + tmp.u32[0] = vec_conversion<uint32_t, uint16_t>((uint16_t)a);
70 + tmp.u32[1] = vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U));
71 + return tmp.u32x2;
72 +}
73 +
74 +// fp8x8 -> half2x4
75 +template <>
76 +__inline__ __device__ uint4 vec_conversion<uint4, uint2>(const uint2& a)
77 +{
78 + union {
79 + uint4 u64x2;
80 + uint2 u64[2];
81 + } tmp;
82 + tmp.u64[0] = vec_conversion<uint2, uint32_t>(a.x);
83 + tmp.u64[1] = vec_conversion<uint2, uint32_t>(a.y);
84 + return tmp.u64x2;
85 +}
86 +
87 +using __nv_bfloat16 = __hip_bfloat16;
88 +
89 +// fp8 -> __nv_bfloat16
90 +template <>
91 +__inline__ __device__ __nv_bfloat16 vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a)
92 +{
93 + hip_fp8 f8{a, hip_fp8::from_bits()};
94 + float f{f8};
95 + return __float2bfloat16(f);
96 +}
97 +
98 +using __nv_bfloat162 = __hip_bfloat162;
99 +
100 +// fp8x2 -> __nv_bfloat162
101 +template <>
102 +__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a)
103 +{
104 + __nv_bfloat162 res;
105 + res.x = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a);
106 + res.y = vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U));
107 + return res;
108 +}
109 +
110 +// fp8x4 -> bf16_4_t
111 +template <>
112 +__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a)
113 +{
114 + bf16_4_t res;
115 + res.x = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a);
116 + res.y = vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U));
117 + return res;
118 +}
119 +
120 +// fp8x8 -> bf16_8_t
121 +template <>
122 +__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, uint2>(const uint2& a)
123 +{
124 + bf16_4_t tmp1, tmp2;
125 + tmp1 = vec_conversion<bf16_4_t, uint32_t>(a.x);
126 + tmp2 = vec_conversion<bf16_4_t, uint32_t>(a.y);
127 + bf16_8_t res;
128 + res.x = tmp1.x;
129 + res.y = tmp1.y;
130 + res.z = tmp2.x;
131 + res.w = tmp2.y;
132 + return res;
133 +}
134 +
135 +// fp8 -> float
136 +template <>
137 +__inline__ __device__ float vec_conversion<float, uint8_t>(const uint8_t& a)
138 +{
139 + hip_fp8 fp8{a, hip_fp8::from_bits()};
140 + return static_cast<float>(fp8);
141 +}
142 +
143 +// fp8x2 -> float2
144 +template <>
145 +__inline__ __device__ float2 vec_conversion<float2, uint16_t>(const uint16_t& a)
146 +{
147 +#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
148 + float2 res;
149 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
150 + res.x = f2[0];
151 + res.y = f2[1];
152 + return res;
153 +#else
154 + float2 res;
155 + res.x = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a));
156 + res.y = vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U));
157 + return res;
158 +#endif
159 +}
160 +
161 +// fp8x4 -> float4
162 +template <>
163 +__inline__ __device__ Float4_ vec_conversion<Float4_, uint32_t>(const uint32_t& a)
164 +{
165 + Float4_ res;
166 + res.x = vec_conversion<float2, uint16_t>((uint16_t)a);
167 + res.y = vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U));
168 + return res;
169 +}
170 +
171 +// fp8x8 -> float8
172 +template <>
173 +__inline__ __device__ Float8_ vec_conversion<Float8_, uint2>(const uint2& a)
174 +{
175 + Float4_ tmp1, tmp2;
176 + tmp1 = vec_conversion<Float4_, uint32_t>(a.x);
177 + tmp2 = vec_conversion<Float4_, uint32_t>(a.y);
178 + Float8_ res;
179 + res.x = tmp1.x;
180 + res.y = tmp1.y;
181 + res.z = tmp2.x;
182 + res.w = tmp2.y;
183 + return res;
184 +}
185 +
186 +// half -> fp8
187 +template <>
188 +__inline__ __device__ uint8_t vec_conversion<uint8_t, uint16_t>(const uint16_t& a)
189 +{
190 + __half_raw tmp;
191 + tmp.x = a;
192 +
193 + hip_fp8 f8{static_cast<float>(tmp.data)};
194 + return f8.data;
195 +}
196 +
197 +// bf16 -> fp8
198 +template <>
199 +__inline__ __device__ uint8_t vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a)
200 +{
201 + hip_fp8 res{__bfloat162float(a)};
202 + return res.data;
203 +}
204 +
205 +// float -> fp8
206 +template <>
207 +__inline__ __device__ uint8_t vec_conversion<uint8_t, float>(const float& a)
208 +{
209 + hip_fp8 f8(a);
210 + return f8.data;
211 +}
212 +
213 +// fp8x4 -> float4
214 +template <>
215 +__inline__ __device__ float4 vec_conversion<float4, uint32_t>(const uint32_t& a)
216 +{
217 + Float4_ tmp = vec_conversion<Float4_, uint32_t>(a);
218 + float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
219 + return res;
220 +}
221 +
222 +// float2 -> half2
223 +template <>
224 +__inline__ __device__ uint32_t vec_conversion<uint32_t, float2>(const float2& a)
225 +{
226 + union {
227 + half2 float16;
228 + uint32_t uint32;
229 + };
230 +
231 + float16 = __float22half2_rn(a);
232 + return uint32;
233 +}
234 +
235 +// Float4 -> half2x2
236 +template <>
237 +__inline__ __device__ uint2 vec_conversion<uint2, Float4_>(const Float4_& a)
238 +{
239 + uint2 b;
240 + float2 val;
241 + val.x = a.x.x;
242 + val.y = a.x.y;
243 + b.x = vec_conversion<uint32_t, float2>(val);
244 +
245 + val.x = a.y.x;
246 + val.y = a.y.y;
247 + b.y = vec_conversion<uint32_t, float2>(val);
248 + return b;
249 +}
250 +
251 +// Float4 -> float4
252 +template <>
253 +__inline__ __device__ float4 vec_conversion<float4, Float4_>(const Float4_& a)
254 +{
255 + float4 b;
256 + b.x = a.x.x;
257 + b.y = a.x.y;
258 + b.z = a.y.x;
259 + b.w = a.y.y;
260 + return b;
261 +}
262 +
263 +// Float8 -> half2x4
264 +template <>
265 +__inline__ __device__ uint4 vec_conversion<uint4, Float8_>(const Float8_& a)
266 +{
267 + uint4 b;
268 + b.x = vec_conversion<uint32_t, float2>(a.x);
269 + b.y = vec_conversion<uint32_t, float2>(a.y);
270 + b.z = vec_conversion<uint32_t, float2>(a.z);
271 + b.w = vec_conversion<uint32_t, float2>(a.w);
272 + return b;
273 +}
274 +
275 +// float2 -> bfloat162
276 +template <>
277 +__inline__ __device__ __nv_bfloat162 vec_conversion<__nv_bfloat162, float2>(const float2& a)
278 +{
279 + __nv_bfloat162 b = __float22bfloat162_rn(a);
280 + return b;
281 +}
282 +
283 +// Float4 -> bfloat162x2
284 +template <>
285 +__inline__ __device__ bf16_4_t vec_conversion<bf16_4_t, Float4_>(const Float4_& a)
286 +{
287 + bf16_4_t b;
288 + b.x = __float22bfloat162_rn(a.x);
289 + b.y = __float22bfloat162_rn(a.y);
290 + return b;
291 +}
292 +
293 +// Float8 -> bfloat162x4
294 +template <>
295 +__inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(const Float8_& a)
296 +{
297 + bf16_8_t b;
298 + b.x = __float22bfloat162_rn(a.x);
299 + b.y = __float22bfloat162_rn(a.y);
300 + b.z = __float22bfloat162_rn(a.z);
301 + b.w = __float22bfloat162_rn(a.w);
302 + return b;
303 +}
304 +
305 +
306 +/* Scaled and vectorized conversions, for data exchange between high and low precision domains
307 +
308 + Convention of the scale in API, e.g: FP8_data = Quantization( High_Precision_data / scale )
309 + s.t.
310 + Quantize(HP / scale) => FP8
311 + Dequant(FP8) * scale => HP
312 +
313 + */
314 +
315 +// fp8 -> half
316 +template <>
317 +__inline__ __device__ uint16_t scaled_vec_conversion<uint16_t, uint8_t>(const uint8_t& a, const float scale)
318 +{
319 + hip_fp8 f8{a, hip_fp8::from_bits()};
320 + __half_raw res;
321 + res.data = static_cast<float>(f8) * scale;
322 + return res.x;
323 +}
324 +
325 +// fp8x2 -> half2
326 +template <>
327 +__inline__ __device__ uint32_t scaled_vec_conversion<uint32_t, uint16_t>(const uint16_t& a, const float scale)
328 +{
329 +#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
330 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
331 + union {
332 + __half2_raw h2r;
333 + uint32_t ui32;
334 + } tmp;
335 + tmp.h2r.x.data = f2[0] * scale;
336 + tmp.h2r.y.data = f2[1] * scale;
337 + return tmp.ui32;
338 +#else
339 + union {
340 + uint16_t u16[2];
341 + uint32_t u32;
342 + } tmp;
343 +
344 + tmp.u16[0] = scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a), scale);
345 + tmp.u16[1] = scaled_vec_conversion<uint16_t, uint8_t>(static_cast<uint8_t>(a >> 8U), scale);
346 + return tmp.u32;
347 +#endif
348 +}
349 +
350 +// fp8x4 -> half2x2
351 +template <>
352 +__inline__ __device__ uint2 scaled_vec_conversion<uint2, uint32_t>(const uint32_t& a, const float scale)
353 +{
354 + union {
355 + uint2 u32x2;
356 + uint32_t u32[2];
357 + } tmp;
358 + tmp.u32[0] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)a, scale);
359 + tmp.u32[1] = scaled_vec_conversion<uint32_t, uint16_t>((uint16_t)(a >> 16U), scale);
360 + return tmp.u32x2;
361 +}
362 +
363 +// fp8x8 -> half2x4
364 +template <>
365 +__inline__ __device__ uint4 scaled_vec_conversion<uint4, uint2>(const uint2& a, const float scale)
366 +{
367 + union {
368 + uint4 u64x2;
369 + uint2 u64[2];
370 + } tmp;
371 + tmp.u64[0] = scaled_vec_conversion<uint2, uint32_t>(a.x, scale);
372 + tmp.u64[1] = scaled_vec_conversion<uint2, uint32_t>(a.y, scale);
373 + return tmp.u64x2;
374 +}
375 +
376 +using __nv_bfloat16 = __hip_bfloat16;
377 +
378 +// fp8 -> __nv_bfloat16
379 +template <>
380 +__inline__ __device__ __nv_bfloat16 scaled_vec_conversion<__nv_bfloat16, uint8_t>(const uint8_t& a, const float scale)
381 +{
382 + hip_fp8 f8{a, hip_fp8::from_bits()};
383 + float f{f8};
384 + return __float2bfloat16(f * scale);
385 +}
386 +
387 +using __nv_bfloat162 = __hip_bfloat162;
388 +
389 +// fp8x2 -> __nv_bfloat162
390 +template <>
391 +__inline__ __device__ __nv_bfloat162 scaled_vec_conversion<__nv_bfloat162, uint16_t>(const uint16_t& a, const float scale)
392 +{
393 + __nv_bfloat162 res;
394 + res.x = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)a, scale);
395 + res.y = scaled_vec_conversion<__nv_bfloat16, uint8_t>((uint8_t)(a >> 8U), scale);
396 + return res;
397 +}
398 +
399 +// fp8x4 -> bf16_4_t
400 +template <>
401 +__inline__ __device__ bf16_4_t scaled_vec_conversion<bf16_4_t, uint32_t>(const uint32_t& a, const float scale)
402 +{
403 + bf16_4_t res;
404 + res.x = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)a, scale);
405 + res.y = scaled_vec_conversion<__nv_bfloat162, uint16_t>((uint16_t)(a >> 16U), scale);
406 + return res;
407 +}
408 +
409 +// fp8x8 -> bf16_8_t
410 +template <>
411 +__inline__ __device__ bf16_8_t scaled_vec_conversion<bf16_8_t, uint2>(const uint2& a, const float scale)
412 +{
413 + bf16_4_t tmp1, tmp2;
414 + tmp1 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.x, scale);
415 + tmp2 = scaled_vec_conversion<bf16_4_t, uint32_t>(a.y, scale);
416 + bf16_8_t res;
417 + res.x = tmp1.x;
418 + res.y = tmp1.y;
419 + res.z = tmp2.x;
420 + res.w = tmp2.y;
421 + return res;
422 +}
423 +
424 +// fp8 -> float
425 +template <>
426 +__inline__ __device__ float scaled_vec_conversion<float, uint8_t>(const uint8_t& a, const float scale)
427 +{
428 + hip_fp8 fp8{a, hip_fp8::from_bits()};
429 + return static_cast<float>(fp8) * scale;
430 +}
431 +
432 +// fp8x2 -> float2
433 +template <>
434 +__inline__ __device__ float2 scaled_vec_conversion<float2, uint16_t>(const uint16_t& a, const float scale)
435 +{
436 +#if defined(__HIP__MI300__) && defined(__HIP_FP8_EXPERIMENTAL_BULK_CONVERT__)
437 + float2 res;
438 + const auto& f2 = __builtin_amdgcn_cvt_pk_f32_fp8(a, 0);
439 + res.x = f2[0] * scale;
440 + res.y = f2[1] * scale;
441 + return res;
442 +#else
443 + float2 res;
444 + res.x = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a), scale);
445 + res.y = scaled_vec_conversion<float, uint8_t>(static_cast<uint8_t>(a >> 8U), scale);
446 + return res;
447 +#endif
448 +}
449 +
450 +// fp8x4 -> float4
451 +template <>
452 +__inline__ __device__ Float4_ scaled_vec_conversion<Float4_, uint32_t>(const uint32_t& a, const float scale)
453 +{
454 + Float4_ res;
455 + res.x = scaled_vec_conversion<float2, uint16_t>((uint16_t)a, scale);
456 + res.y = scaled_vec_conversion<float2, uint16_t>((uint16_t)(a >> 16U), scale);
457 + return res;
458 +}
459 +
460 +// fp8x8 -> float8
461 +template <>
462 +__inline__ __device__ Float8_ scaled_vec_conversion<Float8_, uint2>(const uint2& a, const float scale)
463 +{
464 + Float4_ tmp1, tmp2;
465 + tmp1 = scaled_vec_conversion<Float4_, uint32_t>(a.x, scale);
466 + tmp2 = scaled_vec_conversion<Float4_, uint32_t>(a.y, scale);
467 + Float8_ res;
468 + res.x = tmp1.x;
469 + res.y = tmp1.y;
470 + res.z = tmp2.x;
471 + res.w = tmp2.y;
472 + return res;
473 +}
474 +
475 +
476 +/* Quantize(HP / scale) => FP8 */
477 +
478 +// TODO(Hai): vectorized to add
479 +
480 +// half -> fp8
481 +template <>
482 +__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, uint16_t>(const uint16_t& a, const float scale)
483 +{
484 + __half_raw tmp;
485 + tmp.x = a;
486 +
487 + hip_fp8 f8{static_cast<float>(tmp.data)/scale};
488 + return f8.data;
489 +}
490 +
491 +// bf16 -> fp8
492 +template <>
493 +__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(const __nv_bfloat16& a, const float scale)
494 +{
495 + hip_fp8 res{__bfloat162float(a)/scale};
496 + return res.data;
497 +}
498 +
499 +// float -> fp8
500 +template <>
501 +__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(const float& a, const float scale)
502 +{
503 + hip_fp8 f8(a/scale);
504 + return f8.data;
505 +}
506 +
507 +// fp8x4 -> float4
508 +template <>
509 +__inline__ __device__ float4 scaled_vec_conversion<float4, uint32_t>(const uint32_t& a, const float scale)
510 +{
511 + Float4_ tmp = scaled_vec_conversion<Float4_, uint32_t>(a, scale);
512 + float4 res = make_float4(tmp.x.x, tmp.x.y, tmp.y.x, tmp.y.y);
513 + return res;
514 +}
515 +
516 +}
517 +} // namespace vllm
docs/source/index.rst
+2 -1
@@ -91,7 +91,8 @@ Documentation
91 :caption: Quantization
92
93 quantization/auto_awq
94 - quantization/fp8_e5m2_kv_cache
94 + quantization/fp8_e5m2_kvcache
95 + quantization/fp8_e4m3_kvcache
96
97 .. toctree::
98 :maxdepth: 2
docs/source/quantization/fp8_e4m3_kvcache.rst new
+49
@@ -0,0 +1,49 @@
1 +.. _fp8_e4m3_kvcache:
2 +
3 +FP8 E4M3 KV Cache
4 +==================
5 +
6 +Quantizing the KV cache to FP8 reduces its memory footprint. This increases the number of tokens that can be stored in the cache,
7 +improving throughput. OCP (Open Compute Project www.opencompute.org) specifies two common 8-bit floating point data formats: E5M2
8 +(5 exponent bits and 2 mantissa bits) and E4M3FN (4 exponent bits and 3 mantissa bits), often shortened as E4M3. One benefit of
9 +the E4M3 format over E5M2 is that floating point numbers are represented in higher precision. However, the small dynamic range of
10 +FP8 E4M3 (±240.0 can be represented) typically necessitates the use of a higher-precision (typically FP32) scaling factor alongside
11 +each quantized tensor. For now, only per-tensor (scalar) scaling factors are supported. Development is ongoing to support scaling
12 +factors of a finer granularity (e.g. per-channel).
13 +
14 +These scaling factors can be specified by passing an optional quantization param JSON to the LLM engine at load time. If
15 +this JSON is not specified, scaling factors default to 1.0. These scaling factors are typically obtained when running an
16 +unquantized model through a quantizer tool (e.g. AMD quantizer or NVIDIA AMMO).
17 +
18 +To install AMMO (AlgorithMic Model Optimization):
19 +
20 +.. code-block:: console
21 +
22 + $ pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com nvidia-ammo
23 +
24 +Studies have shown that FP8 E4M3 quantization typically only minimally degrades inference accuracy. The most recent silicon
25 +offerings e.g. AMD MI300, NVIDIA Hopper or later support native hardware conversion to and from fp32, fp16, bf16, etc.
26 +Thus, LLM inference is greatly accelerated with minimal accuracy loss.
27 +
28 +
29 +Here is an example of how to enable this feature:
30 +
31 +.. code-block:: python
32 +
33 + # two float8_e4m3fn kv cache scaling factor files are provided under tests/fp8_kv, please refer to
34 + # https://github.com/vllm-project/vllm/blob/main/examples/fp8/README.md to generate kv_cache_scales.json of your own.
35 +
36 + from vllm import LLM, SamplingParams
37 + sampling_params = SamplingParams(temperature=1.3, top_p=0.8)
38 + llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",
39 + kv_cache_dtype="fp8",
40 + quantization_param_path="./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json")
41 + prompt = "London is the capital of"
42 + out = llm.generate(prompt, sampling_params)[0].outputs[0].text
43 + print(out)
44 +
45 + # output w/ scaling factors: England, the United Kingdom, and one of the world's leading financial,
46 + # output w/o scaling factors: England, located in the southeastern part of the country. It is known
47 +
48 +Note, current prefix caching doesn't work with FP8 KV cache enabled, forward_prefix kernel should handle different KV and cache type.
49 +
docs/source/quantization/fp8_e5m2_kvcache.rst renamed
+5 -2
@@ -1,4 +1,4 @@
1 -.. _fp8_e5m2_kv_cache:
1 +.. _fp8_kv_cache:
2
3 FP8 E5M2 KV Cache
4 ==================
@@ -21,7 +21,7 @@ Here is an example of how to enable this feature:
21 # Create a sampling params object.
22 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
23 # Create an LLM.
24 - llm = LLM(model="facebook/opt-125m", kv_cache_dtype="fp8_e5m2")
24 + llm = LLM(model="facebook/opt-125m", kv_cache_dtype="fp8")
25 # Generate texts from the prompts. The output is a list of RequestOutput objects
26 # that contain the prompt, generated text, and other information.
27 outputs = llm.generate(prompts, sampling_params)
@@ -31,3 +31,6 @@ Here is an example of how to enable this feature:
31 generated_text = output.outputs[0].text
32 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
33
34 +
35 +Note, current prefix caching doesn't work with FP8 KV cache enabled, forward_prefix kernel should handle different KV and cache type.
36 +
examples/fp8/README.md new
+96
@@ -0,0 +1,96 @@
1 +# FP8 KV Cache
2 +
3 +This utility extracts the KV cache scaling factors from a quantized HF (Hugging Face) model. The extracted scaling factors are saved to a JSON file, which can later be used by vLLM (variable-length language model) during runtime. This tool is particularly useful when the KV cache data type is FP8 and is intended for use on ROCm (AMD GPU) platforms.
4 +
5 +## Prerequisites
6 +
7 +- Python 3.x
8 +- PyTorch
9 +- NumPy
10 +- Hugging Face Transformers
11 +- Hugging Face Hub
12 +- AMMO
13 +
14 +Before incorporating the FP8 datatype for inference workloads, you must adhere to the following steps:
15 +1. Install all necessary prerequisites and dependencies.
16 +2. Convert HF model into a quantized HF model.
17 +3. Extract KV Cache Scaling Factors from quantized HF model.
18 +4. Load KV Cache Scaling Factors into VLLM.
19 +
20 +### 2. Convert HF model into a quantized HF model.
21 +Note: The following steps are adapted from the [TensorRT-LLM repository](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/quantization/README.md).
22 +
23 +`quantize.py` (examples/fp8/quantizer/quantize.py) uses the quantization toolkit (AMMO) to calibrate the PyTorch models and export TensorRT-LLM checkpoints. Each TensorRT-LLM checkpoint contains a config file (in .json format) and one or several rank weight files (in .safetensors format).
24 +
25 +The detailed quantization toolkit (AMMO) conversion guide for FP8 can be found at `examples/fp8/quantizer/README.md`.
26 +
27 +### 3. Extract KV Cache Scaling Factors from quantized HF model.
28 +`extract_scales.py` (examples/fp8/extract_scales.py) can be utilized to extract the KV cache scaling factors from your quantized HF model, however at the moment, this tool exclusively supports Llama 2 models. It is also important to note the following:
29 +1. **File Structure**: The utility operates under the assumption that all parameters, including KV cache scaling factors, corresponding to a particular Tensor Parallelism (TP) rank are stored in a single file. These files must adhere to a specific naming convention where the TP rank is immediately identified after a specific keyword (e.g., "rank") in the filename.
30 +
31 +2. **TP Decomposition**: The utility assumes consistency between the TP decomposition employed by the quantizer tool and that used by vLLM.
32 +
33 +3. **AMMO Compatibility**: Currently, the generated KV cache scaling factors for AMMO remain uniform across all TP ranks.
34 +
35 +```python
36 +# prerequisites:
37 +# - Quantized HF LLaMa 2 model
38 +python3 examples/fp8/extract_scales.py --help
39 +Usage: extract_scales.py [-h] --quantized_model QUANTIZED_MODEL [--load_format {auto,safetensors,npz,pt}] [--output_dir OUTPUT_DIR] [--output_name OUTPUT_NAME] [--tp_size TP_SIZE]
40 +
41 +KV Scale Extraction Example
42 +
43 +optional arguments:
44 +--quantized_model: Specify either the local path to, or name of, a quantized HF model. It is expected that the quantization format is FP8_E4M3, for use on ROCm (AMD GPU).
45 +Optional arguments:
46 +--cache_dir: Specify a cache directory to use in the event of a HF model download. (Default: None)
47 +--load_format: Specify the format of the model's tensor files containing the KV cache scaling factors. (Choices: auto, safetensors, npz, pt; Default: auto)
48 +--revision: Specify the model's revision number. (Default: None)
49 +--output_dir: Specify the output directory. By default the KV cache scaling factors will be saved in the model directory. (Default: None)
50 +--output_name: Specify the output filename. (Default: kv_cache_scales.json)
51 +--tp_size: Specify the tensor-parallel (TP) size that the quantized model should correspond to. If specified, during KV cache scaling factor extraction the observed TP size will be checked against this and an error will be raised if there is a mismatch. (Default: None)
52 +```
53 +```python
54 +Example:
55 +python3 examples/fp8/extract_scales.py --quantized_model <QUANTIZED_MODEL_DIR> --tp_size <TENSOR_PARALLEL_SIZE> --output_dir <PATH_TO_OUTPUT_DIR>
56 +```
57 +### 4. Load KV Cache Scaling Factors into VLLM.
58 +This script evaluates the inference throughput of language models using various backends such as vLLM. It measures the time taken to process a given number of prompts and generate sequences for each prompt. The recently generated KV cache scaling factors are now integrated into the benchmarking process and allow for KV cache scaling factors to be utilized for FP8.
59 +```python
60 +# prerequisites:
61 +# - LLaMa 2 kv_cache_scales.json file
62 +
63 +python3 benchmarks/benchmark_throughput.py --help
64 +usage: benchmark_throughput.py [-h] [--backend {vllm,hf,mii}] [--dataset DATASET] [--input-len INPUT_LEN] [--output-len OUTPUT_LEN] [--model MODEL]
65 + [--tokenizer TOKENIZER] [--quantization {awq,gptq,squeezellm,None}] [--tensor-parallel-size TENSOR_PARALLEL_SIZE] [--n N]
66 + [--use-beam-search] [--num-prompts NUM_PROMPTS] [--seed SEED] [--hf-max-batch-size HF_MAX_BATCH_SIZE] [--trust-remote-code]
67 + [--max-model-len MAX_MODEL_LEN] [--dtype {auto,half,float16,bfloat16,float,float32}] [--enforce-eager] [--kv-cache-dtype {auto,fp8}]
68 + [--quantization-param-path KV_CACHE_quantization_param_path]
69 +
70 +Benchmark Throughput Example
71 +optional arguments:
72 + -h, --help show this help message and exit
73 + --backend {vllm,hf,mii}
74 + --dataset DATASET Path to the dataset.
75 + --input-len INPUT_LEN Input prompt length for each request
76 + --output-len OUTPUT_LEN Output length for each request. Overrides the output length from the dataset.
77 + --model MODEL
78 + --tokenizer TOKENIZER
79 + --quantization {awq,gptq,squeezellm,None}, -q {awq,gptq,squeezellm,None}
80 + --tensor-parallel-size TENSOR_PARALLEL_SIZE, -tp TENSOR_PARALLEL_SIZE
81 + --n N Number of generated sequences per prompt.
82 + --use-beam-search
83 + --num-prompts NUM_PROMPTS Number of prompts to process.
84 + --seed SEED
85 + --hf-max-batch-size HF_MAX_BATCH_SIZE Maximum batch size for HF backend.
86 + --trust-remote-code trust remote code from huggingface
87 + --max-model-len MAX_MODEL_LEN Maximum length of a sequence (including prompt and output). If None, will be derived from the model.
88 + --dtype {auto,half,float16,bfloat16,float,float32} data type for model weights and activations. The "auto" option will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models.
89 + --enforce-eager enforce eager execution
90 + --kv-cache-dtype {auto,fp8} Data type for kv cache storage. If "auto", will use model data type. FP8_E5M2 (without scaling) is only supported on cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported ```for common inference criteria.
91 + --quantization-param-path QUANT_PARAM_JSON Path to the JSON file containing the KV cache scaling factors. This should generally be supplied, when KV cache dtype is FP8. Otherwise, KV cache scaling factors default to 1.0, which may cause accuracy issues. FP8_E5M2 (without scaling) is only supported on cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead supported for common inference criteria.
92 +```
93 +```
94 +Example:
95 +python3 benchmarks/benchmark_throughput.py --input-len <INPUT_LEN> --output-len <OUTPUT_LEN> -tp <TENSOR_PARALLEL_SIZE> --kv-cache-dtype fp8 --quantization-param-path <path/to/kv_cache_scales.json> --model <path-to-llama2>
96 +```python
examples/fp8/extract_scales.py new
+367
@@ -0,0 +1,367 @@
1 +import argparse
2 +import glob
3 +import json
4 +import os
5 +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
6 +
7 +import numpy as np
8 +import torch
9 +from safetensors.torch import safe_open
10 +
11 +from vllm.model_executor.layers.quantization.schema import QuantParamSchema
12 +
13 +
14 +# Adapted from vllm/model_executor/weight_utils.py
15 +# The main differences are that we add the NPZ format and simplify
16 +# its functionality drastically for our purposes (e.g. we assume that
17 +# the quantized model exists locally and there is no need to download it)
18 +def _prepare_hf_weights(
19 + quantized_model_dir: str,
20 + load_format: str = "auto",
21 + fall_back_to_pt: bool = True,
22 +) -> Tuple[str, List[str], bool]:
23 + if not os.path.isdir(quantized_model_dir):
24 + raise FileNotFoundError(
25 + f"The quantized model directory `{quantized_model_dir}` "
26 + "does not exist.")
27 + use_safetensors = False
28 + # Some quantized models use .pt files for storing the weights.
29 + if load_format == "auto":
30 + allow_patterns = ["*.safetensors", "*.bin"]
31 + elif load_format == "safetensors":
32 + use_safetensors = True
33 + allow_patterns = ["*.safetensors"]
34 + elif load_format == "pt":
35 + allow_patterns = ["*.pt"]
36 + elif load_format == "npz":
37 + allow_patterns = ["*.npz"]
38 + else:
39 + raise ValueError(f"Unknown load_format: {load_format}")
40 + if fall_back_to_pt:
41 + allow_patterns += ["*.pt"]
42 +
43 + hf_weights_files: List[str] = []
44 + for pattern in allow_patterns:
45 + hf_weights_files += glob.glob(
46 + os.path.join(quantized_model_dir, pattern))
47 + if len(hf_weights_files) > 0:
48 + if pattern == "*.safetensors":
49 + use_safetensors = True
50 + break
51 +
52 + if not use_safetensors:
53 + # Exclude files that are not needed for inference.
54 + # https://github.com/huggingface/transformers/blob/v4.34.0/src/transformers/trainer.py#L227-L233
55 + blacklist = [
56 + "training_args.bin",
57 + "optimizer.bin",
58 + "optimizer.pt",
59 + "scheduler.pt",
60 + "scaler.pt",
61 + ]
62 + hf_weights_files = [
63 + f for f in hf_weights_files
64 + if not any(f.endswith(x) for x in blacklist)
65 + ]
66 +
67 + if len(hf_weights_files) == 0:
68 + raise RuntimeError(
69 + f"Cannot find any model weights with `{quantized_model_dir}`")
70 +
71 + return hf_weights_files, use_safetensors
72 +
73 +
74 +# Adapted from vllm/model_executor/weight_utils.py
75 +def _hf_tensorfile_iterator(filename: str, load_format: str,
76 + use_safetensors: bool):
77 + if load_format == "npz":
78 + assert not use_safetensors
79 + with np.load(filename) as data:
80 + for name in data.files:
81 + param = torch.from_numpy(data[name])
82 + yield name, param
83 + elif use_safetensors:
84 + with safe_open(filename, framework="pt") as f:
85 + for name in f.keys(): # NOQA: SIM118
86 + param = f.get_tensor(name)
87 + yield name, param
88 + else:
89 + state = torch.load(filename, map_location="cpu")
90 + for name, param in state.items():
91 + yield name, param
92 + del state
93 + torch.cuda.empty_cache()
94 +
95 +
96 +def _kv_scales_extractor(
97 + hf_tensor_files: Iterable[str],
98 + use_safetensors: bool,
99 + rank_keyword: str = "rank",
100 + expected_tp_size: Optional[int] = None) -> Dict[int, Dict[int, float]]:
101 + """
102 + Given a list of files containing tensor data, attempt to extract KV cache
103 + scales from these files. Intended as a helper function taking in the output
104 + from _prepare_hf_weights.
105 + Args:
106 + rank_keyword Matches the number immediately after this keyword in the
107 + tensor filename to determine the TP rank corresponding
108 + to said tensor file
109 + expected_tp_size If specified, the TP size of the tensor files is checked
110 + against this and an error is raised if they don't match.
111 + Returns a dictionary mapping TP ranks to their relevant KV cache scales.
112 + The per-rank scales are themselves represented as a dictionary of layer
113 + indices to the respective per-layer scale.
114 + """
115 + for char in rank_keyword:
116 + assert not char.isdecimal(
117 + ), f"Rank keyword {rank_keyword} contains a numeric character!"
118 + rank_scales_map = {}
119 + for tensor_file in hf_tensor_files:
120 + try:
121 + rank_idx = tensor_file.find(rank_keyword)
122 + if rank_idx != -1:
123 + start_idx = rank_idx + len(rank_keyword)
124 + stop_idx = start_idx
125 + while stop_idx < len(
126 + tensor_file) and tensor_file[stop_idx].isdecimal():
127 + stop_idx += 1
128 + if stop_idx == start_idx:
129 + raise RuntimeError("Did not find rank # in filename.")
130 + rank = int(tensor_file[start_idx:stop_idx])
131 + elif len(hf_tensor_files) == 1:
132 + # Since there is only one tensor file, we can assume
133 + # that it's intended for TP rank 0
134 + rank = 0
135 + else:
136 + raise RuntimeError(
137 + f"Filename does not contain '{rank_keyword}'.")
138 + except RuntimeError:
139 + print("Unable to determine TP rank "
140 + f"corresponding to file '{tensor_file}'")
141 + raise
142 +
143 + if rank not in rank_scales_map:
144 + layer_scales_map = {}
145 + rank_scales_map[rank] = layer_scales_map
146 + else:
147 + raise RuntimeError(
148 + f"Tensor file '{tensor_file}' shares TP rank {rank} "
149 + "with another tensor file.")
150 +
151 + module_delimiter = ":" if args.load_format == "npz" else "."
152 + for name, param in _hf_tensorfile_iterator(tensor_file,
153 + args.load_format,
154 + use_safetensors):
155 + if "kv_cache_scaling_factor" in name:
156 + nums = [
157 + int(s) for s in name.split(module_delimiter)
158 + if s.isdecimal()
159 + ]
160 + assert len(
161 + nums) == 1, f"Could not determine layer idx for {name}"
162 + layer_idx = nums[0]
163 + assert layer_idx not in layer_scales_map, f"Duplicate scaling"\
164 + f" factor corresponding to layer {layer_idx}"
165 + try:
166 + layer_scales_map[layer_idx] = param.item()
167 + except RuntimeError:
168 + print(
169 + "This utility supports only per-tensor scalar scales "
170 + f"for now. The tensor\n {name} = {param} \nis an "
171 + "invalid scale factor.")
172 + raise
173 +
174 + if all(
175 + len(layer_scales_map) == 0
176 + for layer_scales_map in rank_scales_map.values()):
177 + # Note: this is true even if the rank_scales_map is empty
178 + print("WARNING: No KV cache scale factors found. No output saved.")
179 + return None
180 + empirical_tp_world_size = max(rank_scales_map.keys()) + 1
181 + if expected_tp_size is not None:
182 + assert expected_tp_size == empirical_tp_world_size, \
183 + f"User expected TP world size = {expected_tp_size} " \
184 + "from model but tool is expecting TP world size = " \
185 + f"{empirical_tp_world_size} from model instead."
186 + for i in range(empirical_tp_world_size):
187 + assert i in rank_scales_map, "Expected TP world size = "\
188 + f"{empirical_tp_world_size} but did not find KV " \
189 + f"cache scaling factors for TP rank {i}"
190 + print(f"Found TP world size = {empirical_tp_world_size} "
191 + "when extracting KV cache scales!")
192 + return rank_scales_map
193 +
194 +
195 +def _metadata_extractor(quantized_model_dir: str,
196 + metadata_extract_fns: \
197 + Dict[str, Callable[[Dict[str, Any]], Any]]) \
198 + -> Dict[str, Any]:
199 + """
200 + Given a directory containing quantized model files, this function
201 + aims to extract metadata from the JSON files within this directory.
202 + Each JSON file is expected to represent a dictionary in JSON
203 + format (referred to as a "JSON-dictionary"). Metadata extraction is
204 + defined by a dictionary called metadata_extract_fns, where each
205 + metadata field name is mapped to an extraction function.
206 +
207 + These extraction functions are designed to take a JSON-dictionary
208 + as their only argument and return the corresponding metadata.
209 + While extraction functions are permitted to raise exceptions, they
210 + should only raise a KeyError or ValueError if the metadata field
211 + cannot be extracted from the current JSON-dictionary, yet there's
212 + a possibility of finding it in another JSON-dictionary.
213 +
214 + The function returns a dictionary that maps metadata fields to
215 + their extracted data. The keys of this dictionary correspond exactly
216 + to those in metadata_extract_fns. If any fields fail to be extracted,
217 + their corresponding values are set to None, and a warning is printed.
218 + """
219 + if not os.path.isdir(quantized_model_dir):
220 + raise FileNotFoundError(
221 + f"The quantized model directory `{quantized_model_dir}` "
222 + "does not exist.")
223 + metadata_files = glob.glob(os.path.join(quantized_model_dir, "*.json"))
224 +
225 + result = {}
226 + for file in metadata_files:
227 + with open(file) as f:
228 + try:
229 + metadata = json.load(f)
230 + except json.JSONDecodeError:
231 + print(f"Could not parse `{file}` as a valid metadata file,"
232 + " skipping it.")
233 + continue
234 + if not isinstance(metadata, dict):
235 + print(f"The file `{file}` does not correspond to a "
236 + "JSON-serialized dictionary, skipping it.")
237 + continue
238 + for metadata_name, extract_fn in metadata_extract_fns.items():
239 + try:
240 + metadata_info = extract_fn(metadata)
241 + if metadata_name not in result:
242 + result[metadata_name] = metadata_info
243 + elif metadata_info != result[metadata_name]:
244 + raise RuntimeError(
245 + "Metadata mismatch! Originally found "
246 + f"{metadata_name} = {result[metadata_name]} but "
247 + f"now found {metadata_name} = {metadata_info} in "
248 + f"`{file}`")
249 + except KeyError:
250 + # It is possible that a given file does not contain some
251 + # of our selected metadata as it could be located in some
252 + # other metadata file.
253 + # 'EFINAE': extract_fn failure is not an error.
254 + pass
255 + except ValueError:
256 + # See above.
257 + pass
258 +
259 + # Warn if we cannot find any of the requested metadata
260 + for metadata_name in metadata_extract_fns:
261 + if metadata_name not in result:
262 + print("WARNING: Unable to find requested metadata field "
263 + f"`{metadata_name}`, setting it to None.")
264 + result[metadata_name] = None
265 +
266 + return result
267 +
268 +
269 +def main(args):
270 + metadata_extract_fns = {
271 + "model_type": lambda json_dict: json_dict["layers"][0]["decoder_type"],
272 + "tp_size": lambda json_dict: int(json_dict["tensor_parallel"]),
273 + "model_dtype": lambda json_dict: json_dict["dtype"]
274 + }
275 + recovered_metadata = _metadata_extractor(args.quantized_model,
276 + metadata_extract_fns)
277 + if args.tp_size is not None:
278 + metadata_tp_size = recovered_metadata["tp_size"]
279 + if metadata_tp_size is not None:
280 + assert args.tp_size == metadata_tp_size, \
281 + f"User expected TP world size = {args.tp_size} " \
282 + f"but found TP world size = {metadata_tp_size} from metadata!"
283 + expected_tp_size = args.tp_size or recovered_metadata["tp_size"]
284 + rank_keyword = "rank"
285 + hf_tensor_files, use_safetensors = _prepare_hf_weights(
286 + args.quantized_model, args.load_format)
287 + rank_scales_map = _kv_scales_extractor(hf_tensor_files, use_safetensors,
288 + rank_keyword, expected_tp_size)
289 + # Postprocess: formatting to the current schema. Consider pulling it
290 + # out into a dedicated function should it ever become more complicated.
291 + rank_scales_map = {
292 + rank: {k: scale[k]
293 + for k in sorted(scale.keys())}
294 + for rank, scale in rank_scales_map.items()
295 + }
296 + # TODO: Expand this with activation and weights scaling factors when
297 + # they are used in the future
298 + schema = QuantParamSchema(
299 + model_type=recovered_metadata["model_type"],
300 + kv_cache={
301 + "dtype": ("float8_e4m3fn" if len(rank_scales_map) > 0 else
302 + recovered_metadata["model_dtype"]),
303 + "scaling_factor":
304 + rank_scales_map
305 + },
306 + )
307 +
308 + if args.output_dir is None:
309 + output_file = os.path.join(args.quantized_model, args.output_name)
310 + else:
311 + if not os.path.isdir(args.output_dir):
312 + os.makedirs(args.output_dir, exist_ok=True)
313 + output_file = os.path.join(args.output_dir, args.output_name)
314 +
315 + with open(output_file, 'w') as f:
316 + f.write(schema.model_dump_json(indent=4))
317 + print(f"Completed! KV cache scaling factors saved to {output_file}")
318 +
319 +
320 +if __name__ == "__main__":
321 + parser = argparse.ArgumentParser(
322 + description="This simple utility extracts the "
323 + "KV cache scaling factors from a quantized HF model "
324 + "and saves them to a JSON file compatible with later "
325 + "use by vLLM (pass this file to the appropriate "
326 + "runtime typically using the argument "
327 + "--quantization-param-path <filename>). This is only used "
328 + "if the KV cache dtype is FP8 and on ROCm (AMD GPU).")
329 + parser.add_argument(
330 + "--quantized_model",
331 + help="Specify the directory containing a single quantized HF model. "
332 + "It is expected that the quantization format is FP8_E4M3, for use "
333 + "on ROCm (AMD GPU).",
334 + required=True)
335 + parser.add_argument(
336 + "--load_format",
337 + help="Optionally specify the format of the model's tensor files "
338 + "containing the KV cache scaling factors.",
339 + choices=["auto", "safetensors", "npz", "pt"],
340 + default="auto")
341 + parser.add_argument(
342 + "--output_dir",
343 + help="Optionally specify the output directory. By default the "
344 + "KV cache scaling factors will be saved in the model directory, "
345 + "however you can override this behavior here.",
346 + default=None)
347 + parser.add_argument(
348 + "--output_name",
349 + help="Optionally specify the output filename.",
350 + # TODO: Change this once additional scaling factors are enabled
351 + default="kv_cache_scales.json")
352 + parser.add_argument(
353 + "--tp_size",
354 + help="Optionally specify the tensor-parallel (TP) size that the "
355 + "quantized model should correspond to. If specified, during KV "
356 + "cache scaling factor extraction the observed TP size will be "
357 + "checked against this and an error will be raised if there is "
358 + "a mismatch. If not specified, the quantized model's expected "
359 + "TP size is instead inferred from the largest TP rank observed. "
360 + "The expected TP size is cross-checked against the TP ranks "
361 + "observed in the quantized model and an error is raised if any "
362 + "discrepancies are found.",
363 + default=None,
364 + type=int)
365 + args = parser.parse_args()
366 +
367 + main(args)
examples/fp8/quantizer/README.md new
+32
@@ -0,0 +1,32 @@
1 +### Quantizer Utilities
2 +`quantize.py`: NVIDIA Quantization utilities using AMMO, ported from TensorRT-LLM:
3 +`https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/quantization/quantize.py`
4 +
5 +### Prerequisite
6 +
7 +#### AMMO (AlgorithMic Model Optimization) Installation: nvidia-ammo 0.7.1 or later
8 +`pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com nvidia-ammo`
9 +
10 +#### AMMO Download (code and docs)
11 +`https://developer.nvidia.com/downloads/assets/cuda/files/nvidia-ammo/nvidia_ammo-0.5.0.tar.gz`
12 +`https://developer.nvidia.com/downloads/assets/cuda/files/nvidia-ammo/nvidia_ammo-0.7.1.tar.gz`
13 +
14 +### Usage
15 +
16 +#### Run on H100 system for speed if FP8; number of GPUs depends on the model size
17 +
18 +#### Example: quantize Llama2-7b model from HF to FP8 with FP8 KV Cache:
19 +`python quantize.py --model_dir ./ll2-7b --dtype float16 --qformat fp8 --kv_cache_dtype fp8 --output_dir ./ll2_7b_fp8 --calib_size 512 --tp_size 1`
20 +
21 +Outputs: model structure, quantized model & parameters (with scaling factors) are in JSON and Safetensors (npz is generated only for the reference)
22 +```
23 +# ll ./ll2_7b_fp8/
24 +total 19998244
25 +drwxr-xr-x 2 root root 4096 Feb 7 01:08 ./
26 +drwxrwxr-x 8 1060 1061 4096 Feb 7 01:08 ../
27 +-rw-r--r-- 1 root root 176411 Feb 7 01:08 llama_tp1.json
28 +-rw-r--r-- 1 root root 13477087480 Feb 7 01:09 llama_tp1_rank0.npz
29 +-rw-r--r-- 1 root root 7000893272 Feb 7 01:08 rank0.safetensors
30 +#
31 +```
32 +
examples/fp8/quantizer/quantize.py new
+369
@@ -0,0 +1,369 @@
1 +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2 +# SPDX-License-Identifier: Apache-2.0
3 +#
4 +# Licensed under the Apache License, Version 2.0 (the "License");
5 +# you may not use this file except in compliance with the License.
6 +# You may obtain a copy of the License at
7 +#
8 +# http://www.apache.org/licenses/LICENSE-2.0
9 +#
10 +# Unless required by applicable law or agreed to in writing, software
11 +# distributed under the License is distributed on an "AS IS" BASIS,
12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 +# See the License for the specific language governing permissions and
14 +# limitations under the License.
15 +"""
16 +Adapted from examples/quantization/hf_ptq.py
17 +"""
18 +
19 +import argparse
20 +import copy
21 +import json
22 +import random
23 +import time
24 +
25 +import ammo.torch.quantization as atq
26 +import numpy as np
27 +import torch
28 +from ammo.torch.export import export_model_config
29 +from datasets import load_dataset
30 +from torch.utils.data import DataLoader
31 +from transformers import AutoModelForCausalLM, AutoTokenizer
32 +
33 +RAND_SEED = 1234
34 +MAX_SEQ_LEN = 2048
35 +
36 +EMPTY_CFG = {
37 + "quant_cfg": {
38 + "*weight_quantizer": {
39 + "enable": False,
40 + },
41 + "*input_quantizer": {
42 + "enable": False
43 + },
44 + "*lm_head*": {
45 + "enable": False
46 + },
47 + "*output_layer*": {
48 + "enable": False
49 + },
50 + "default": {
51 + "enable": False
52 + },
53 + },
54 + "algorithm": "max",
55 +}
56 +
57 +KV_CACHE_CFG = {
58 + "*.query_key_value.output_quantizer": {
59 + "num_bits": 8,
60 + "axis": None,
61 + "enable": True
62 + },
63 + "*.Wqkv.output_quantizer": {
64 + "num_bits": 8,
65 + "axis": None,
66 + "enable": True
67 + },
68 + "*.W_pack.output_quantizer": {
69 + "num_bits": 8,
70 + "axis": None,
71 + "enable": True
72 + },
73 + "*.c_attn.output_quantizer": {
74 + "num_bits": 8,
75 + "axis": None,
76 + "enable": True
77 + },
78 + "*.k_proj.output_quantizer": {
79 + "num_bits": 8,
80 + "axis": None,
81 + "enable": True
82 + },
83 + "*.v_proj.output_quantizer": {
84 + "num_bits": 8,
85 + "axis": None,
86 + "enable": True
87 + },
88 +}
89 +
90 +QUANT_CFG_CHOICES = {
91 + "int8_sq": atq.INT8_SMOOTHQUANT_CFG,
92 + "fp8": atq.FP8_DEFAULT_CFG,
93 + "int4_awq": atq.INT4_AWQ_CFG,
94 + "w4a8_awq": atq.W4A8_AWQ_BETA_CFG,
95 + "int8_wo": EMPTY_CFG,
96 + "int4_wo": EMPTY_CFG,
97 + "full_prec": EMPTY_CFG,
98 +}
99 +
100 +MODEL_NAME_PATTERN_MAP = {
101 + "GPT2": "gpt2",
102 + "Xverse": "llama",
103 + "Llama": "llama",
104 + "Mistral": "llama",
105 + "GPTJ": "gptj",
106 + "FalconForCausalLM": "falcon",
107 + "RWForCausalLM": "falcon",
108 + "baichuan": "baichuan",
109 + "MPT": "mpt",
110 + "Bloom": "bloom",
111 + "ChatGLM": "chatglm",
112 + "QWen": "qwen",
113 +}
114 +
115 +
116 +def get_tokenizer(ckpt_path, max_seq_len=MAX_SEQ_LEN, model_type=None):
117 + print(f"Initializing tokenizer from {ckpt_path}")
118 + tokenizer = AutoTokenizer.from_pretrained(
119 + ckpt_path,
120 + model_max_length=max_seq_len,
121 + padding_side="left",
122 + trust_remote_code=True,
123 + )
124 + if model_type and model_type == "qwen":
125 + # qwen use token id 151643 as pad and eos tokens
126 + tokenizer.pad_token = tokenizer.convert_ids_to_tokens(151643)
127 + tokenizer.eos_token = tokenizer.convert_ids_to_tokens(151643)
128 +
129 + # can't set attribute 'pad_token' for "<unk>"
130 + if tokenizer.pad_token != "<unk>":
131 + tokenizer.pad_token = tokenizer.eos_token
132 + if tokenizer.pad_token is None:
133 + tokenizer.pad_token = tokenizer.eos_token
134 + assert tokenizer.pad_token is not None, f"Pad token for {model_type} cannot be set!"
135 +
136 + return tokenizer
137 +
138 +
139 +def get_model(ckpt_path, dtype="fp16", device="cuda"):
140 + print(f"Initializing model from {ckpt_path}")
141 + if dtype == "bf16" or dtype == "bfloat16":
142 + dtype = torch.bfloat16
143 + elif dtype == "fp16" or dtype == "float16":
144 + dtype = torch.float16
145 + elif dtype == "fp32" or dtype == "float32":
146 + dtype = torch.float32
147 + else:
148 + raise NotImplementedError(f"Unknown dtype {dtype}")
149 +
150 + # model_kwargs = {"torch_dtype": dtype}
151 + model_kwargs = {"torch_dtype": "auto"}
152 +
153 + model = AutoModelForCausalLM.from_pretrained(ckpt_path,
154 + device_map="auto",
155 + **model_kwargs,
156 + trust_remote_code=True)
157 + model.eval()
158 +
159 + model_dtype = next(model.parameters()).dtype
160 + if dtype != model_dtype:
161 + print(
162 + f"[TensorRT-LLM][WARNING] The manually set model data type is {dtype}, "
163 + f"but the data type of the HuggingFace model is {model_dtype}.")
164 +
165 + return model
166 +
167 +
168 +def get_model_type(model):
169 + for k, v in MODEL_NAME_PATTERN_MAP.items():
170 + if k.lower() in type(model).__name__.lower():
171 + return v
172 + return None
173 +
174 +
175 +def get_calib_dataloader(data="cnn_dailymail",
176 + tokenizer=None,
177 + batch_size=1,
178 + calib_size=512,
179 + block_size=512,
180 + device=None):
181 + print("Loading calibration dataset")
182 + if data == "pileval":
183 + dataset = load_dataset(
184 + "json",
185 + data_files="https://the-eye.eu/public/AI/pile/val.jsonl.zst",
186 + split="train")
187 + dataset = dataset["text"][:calib_size]
188 + elif data == "cnn_dailymail":
189 + dataset = load_dataset("cnn_dailymail", name="3.0.0", split="train")
190 + dataset = dataset["article"][:calib_size]
191 + else:
192 + raise NotImplementedError
193 +
194 + batch_encoded = tokenizer.batch_encode_plus(dataset,
195 + return_tensors="pt",
196 + padding="max_length",
197 + truncation=True,
198 + max_length=block_size)
199 + if device:
200 + batch_encoded = batch_encoded.to(device)
201 + batch_encoded = batch_encoded["input_ids"]
202 +
203 + calib_dataloader = DataLoader(batch_encoded,
204 + batch_size=batch_size,
205 + shuffle=False)
206 +
207 + return calib_dataloader
208 +
209 +
210 +def quantize_model(model, quant_cfg, calib_dataloader=None):
211 +
212 + def calibrate_loop():
213 + if calib_dataloader is None:
214 + return
215 + """Adjusts weights and scaling factors based on selected algorithms."""
216 + for idx, data in enumerate(calib_dataloader):
217 + print(f"Calibrating batch {idx}")
218 + model(data)
219 +
220 + print("Starting quantization...")
221 + start_time = time.time()
222 + atq.quantize(model, quant_cfg, forward_loop=calibrate_loop)
223 + end_time = time.time()
224 + print("Quantization done. Total time used: {:.2f} s.".format(end_time -
225 + start_time))
226 +
227 + return model
228 +
229 +
230 +def main(args):
231 + if not torch.cuda.is_available():
232 + raise EnvironmentError("GPU is required for inference.")
233 +
234 + random.seed(RAND_SEED)
235 + np.random.seed(RAND_SEED)
236 +
237 + model = get_model(args.model_dir, args.dtype, args.device)
238 + model_type = get_model_type(model)
239 + tokenizer = get_tokenizer(args.model_dir, model_type=model_type)
240 +
241 + if args.qformat in ["full_prec", "int8_wo", "int4_wo"
242 + ] and args.kv_cache_dtype is None:
243 + print(f"No quantization applied, export {args.dtype} model")
244 + else:
245 + if "awq" in args.qformat:
246 + if args.calib_size > 32:
247 + print(
248 + f"AWQ calibration could take longer with calib_size = {args.calib_size}, Using"
249 + " calib_size=32 instead")
250 + args.calib_size = 32
251 + print(
252 + "\nAWQ calibration could take longer than other calibration methods. Please"
253 + " increase the batch size to speed up the calibration process. Batch size can be"
254 + " set by adding the argument --batch_size <batch_size> to the command line.\n"
255 + )
256 +
257 + calib_dataloader = get_calib_dataloader(
258 + tokenizer=tokenizer,
259 + batch_size=args.batch_size,
260 + calib_size=args.calib_size,
261 + device=args.device,
262 + )
263 +
264 + if args.qformat in QUANT_CFG_CHOICES:
265 + quant_cfg = QUANT_CFG_CHOICES[args.qformat]
266 + else:
267 + raise ValueError(
268 + f"Unsupported quantization format: {args.qformat}")
269 +
270 + if "awq" in args.qformat:
271 + quant_cfg = copy.deepcopy(QUANT_CFG_CHOICES[args.qformat])
272 + weight_quantizer = quant_cfg["quant_cfg"][
273 + "*weight_quantizer"] # type: ignore
274 + if isinstance(weight_quantizer, list):
275 + weight_quantizer = weight_quantizer[0]
276 + weight_quantizer["block_sizes"][-1] = args.awq_block_size
277 +
278 + if args.kv_cache_dtype is not None:
279 + if args.kv_cache_dtype == "fp8":
280 + for value in KV_CACHE_CFG.values():
281 + value.update({"num_bits": (4, 3)}) # type: ignore
282 + quant_cfg["quant_cfg"].update(KV_CACHE_CFG) # type: ignore
283 +
284 + print(quant_cfg)
285 +
286 + model = quantize_model(model, quant_cfg, calib_dataloader)
287 +
288 + with torch.inference_mode():
289 + if model_type is None:
290 + print(
291 + f"Unknown model type {type(model).__name__}. Continue exporting..."
292 + )
293 + model_type = f"unknown:{type(model).__name__}"
294 +
295 + export_path = args.output_dir
296 + start_time = time.time()
297 +
298 + if args.qformat == "int4_awq" and model_type == "qwen":
299 + torch.save(model.state_dict(), export_path)
300 + else:
301 + export_npz = (model_type not in [
302 + 'gptj', 'falcon', 'chatglm', 'mpt', 'llama', 'baichuan'
303 + ])
304 +
305 + # export safetensors
306 + export_model_config(
307 + model,
308 + model_type,
309 + getattr(torch, args.dtype),
310 + export_dir=export_path,
311 + inference_tensor_parallel=args.tp_size,
312 + inference_pipeline_parallel=args.pp_size,
313 + # export_tensorrt_llm_config=(not export_npz),
314 + export_tensorrt_llm_config=False,
315 + export_npz=export_npz)
316 +
317 + # Workaround for wo quantization
318 + if args.qformat in ["int8_wo", "int4_wo", "full_prec"]:
319 + with open(f"{export_path}/config.json", 'r') as f:
320 + tensorrt_llm_config = json.load(f)
321 + if args.qformat == "int8_wo":
322 + tensorrt_llm_config["quantization"]["quant_algo"] = 'W8A16'
323 + elif args.qformat == "int4_wo":
324 + tensorrt_llm_config["quantization"]["quant_algo"] = 'W4A16'
325 + else:
326 + tensorrt_llm_config["quantization"]["quant_algo"] = None
327 + with open(f"{export_path}/config.json", "w") as f:
328 + json.dump(tensorrt_llm_config, f, indent=4)
329 +
330 + end_time = time.time()
331 + print("Quantized model exported to {} \nTotal time used {:.2f} s.".
332 + format(export_path, end_time - start_time))
333 +
334 +
335 +if __name__ == "__main__":
336 + parser = argparse.ArgumentParser(description=__doc__)
337 + parser.add_argument("--model_dir",
338 + help="Specify where the HuggingFace model is",
339 + required=True)
340 + parser.add_argument("--device", default="cuda")
341 + parser.add_argument("--dtype", help="Model data type.", default="float16")
342 + parser.add_argument(
343 + "--qformat",
344 + help="Quantization format.",
345 + default="full_prec",
346 + choices=[
347 + "fp8", "int8_sq", "int4_awq", "w4a8_awq", "int8_wo", "int4_wo",
348 + "full_prec"
349 + ],
350 + )
351 + parser.add_argument("--batch_size",
352 + help="Batch size for calibration.",
353 + type=int,
354 + default=1)
355 + parser.add_argument("--calib_size",
356 + help="Number of samples for calibration.",
357 + type=int,
358 + default=512)
359 + parser.add_argument("--output_dir", default="exported_model")
360 + parser.add_argument("--tp_size", type=int, default=1)
361 + parser.add_argument("--pp_size", type=int, default=1)
362 + parser.add_argument("--awq_block_size", type=int, default=128)
363 + parser.add_argument("--kv_cache_dtype",
364 + help="KV Cache dtype.",
365 + default=None,
366 + choices=["int8", "fp8", None])
367 + args = parser.parse_args()
368 +
369 + main(args)
pyproject.toml
+4
@@ -13,6 +13,10 @@ build-backend = "setuptools.build_meta"
13 [tool.ruff]
14 # Allow lines to be as long as 80.
15 line-length = 80
16 +exclude = [
17 + # External file, leaving license intact
18 + "examples/fp8/quantizer/quantize.py"
19 +]
20
21 [tool.ruff.lint]
22 select = [
tests/fp8_kv/llama2-70b-fp8-kv/kv_cache_scales.json new
+90
@@ -0,0 +1,90 @@
1 +{
2 + "model_type": "llama",
3 + "kv_cache": {
4 + "dtype": "float8_e4m3fn",
5 + "scaling_factor": {
6 + "0": {
7 + "0": 0.0230364128947258,
8 + "1": 0.01979283057153225,
9 + "2": 0.0241350457072258,
10 + "3": 0.0308314748108387,
11 + "4": 0.0430733822286129,
12 + "5": 0.0370396226644516,
13 + "6": 0.0306222103536129,
14 + "7": 0.0357491634786129,
15 + "8": 0.0358189195394516,
16 + "9": 0.0443289652466774,
17 + "10": 0.0433175228536129,
18 + "11": 0.0416782945394516,
19 + "12": 0.0366908498108387,
20 + "13": 0.0432477705180645,
21 + "14": 0.0410505048930645,
22 + "15": 0.0457589291036129,
23 + "16": 0.0418526791036129,
24 + "17": 0.0432477705180645,
25 + "18": 0.0469447560608387,
26 + "19": 0.0514787957072258,
27 + "20": 0.0541294664144516,
28 + "21": 0.0587681382894516,
29 + "22": 0.0625,
30 + "23": 0.0585588738322258,
31 + "24": 0.0600237175822258,
32 + "25": 0.0588030144572258,
33 + "26": 0.0531180277466774,
34 + "27": 0.06396484375,
35 + "28": 0.0603027381002903,
36 + "29": 0.0582101047039032,
37 + "30": 0.0625348836183548,
38 + "31": 0.0585588738322258,
39 + "32": 0.0582798570394516,
40 + "33": 0.0575125589966774,
41 + "34": 0.0590820349752903,
42 + "35": 0.0614188089966774,
43 + "36": 0.0631975457072258,
44 + "37": 0.0615931935608387,
45 + "38": 0.0601283498108387,
46 + "39": 0.0571986623108387,
47 + "40": 0.0670340433716774,
48 + "41": 0.0523507259786129,
49 + "42": 0.0547223798930645,
50 + "43": 0.0631975457072258,
51 + "44": 0.0663713738322258,
52 + "45": 0.0603376142680645,
53 + "46": 0.0652204304933548,
54 + "47": 0.0734514519572258,
55 + "48": 0.0693708211183548,
56 + "49": 0.0725446492433548,
57 + "50": 0.0627790242433548,
58 + "51": 0.0691266804933548,
59 + "52": 0.0688825398683548,
60 + "53": 0.068429134786129,
61 + "54": 0.0605119988322258,
62 + "55": 0.0799386203289032,
63 + "56": 0.0853097140789032,
64 + "57": 0.0661969929933548,
65 + "58": 0.0689871683716774,
66 + "59": 0.0724051371216774,
67 + "60": 0.0541643425822258,
68 + "61": 0.0626743882894516,
69 + "62": 0.0628487765789032,
70 + "63": 0.0607212632894516,
71 + "64": 0.0589076466858387,
72 + "65": 0.0451660193502903,
73 + "66": 0.0453055277466774,
74 + "67": 0.0414341539144516,
75 + "68": 0.0385044664144516,
76 + "69": 0.0414341539144516,
77 + "70": 0.0466308631002903,
78 + "71": 0.0399693101644516,
79 + "72": 0.0437011756002903,
80 + "73": 0.0434221550822258,
81 + "74": 0.0428989976644516,
82 + "75": 0.0401785746216774,
83 + "76": 0.0431082621216774,
84 + "77": 0.0484444759786129,
85 + "78": 0.0417829267680645,
86 + "79": 0.0418178029358387
87 + }
88 + }
89 + }
90 +}
\ No newline at end of file
tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json new
+42
@@ -0,0 +1,42 @@
1 +{
2 + "model_type": "llama",
3 + "kv_cache": {
4 + "dtype": "float8_e4m3fn",
5 + "scaling_factor": {
6 + "0": {
7 + "0": 0.0152239128947258,
8 + "1": 0.0188860222697258,
9 + "2": 0.0354178324341774,
10 + "3": 0.0376674123108387,
11 + "4": 0.0418526791036129,
12 + "5": 0.0433175228536129,
13 + "6": 0.0397600457072258,
14 + "7": 0.0424455925822258,
15 + "8": 0.0415387861430645,
16 + "9": 0.0408412404358387,
17 + "10": 0.0395856611430645,
18 + "11": 0.0377371683716774,
19 + "12": 0.0400739423930645,
20 + "13": 0.040771484375,
21 + "14": 0.0393415205180645,
22 + "15": 0.0369001142680645,
23 + "16": 0.03857421875,
24 + "17": 0.0387486070394516,
25 + "18": 0.0403180830180645,
26 + "19": 0.0396205373108387,
27 + "20": 0.0375627800822258,
28 + "21": 0.0407366082072258,
29 + "22": 0.0432477705180645,
30 + "23": 0.0377022884786129,
31 + "24": 0.0399693101644516,
32 + "25": 0.0374581478536129,
33 + "26": 0.0413295216858387,
34 + "27": 0.0442243330180645,
35 + "28": 0.0424804724752903,
36 + "29": 0.0456891767680645,
37 + "30": 0.0409109964966774,
38 + "31": 0.0482352152466774
39 + }
40 + }
41 + }
42 +}
tests/kernels/test_attention.py
+11 -5
@@ -32,7 +32,7 @@ HEAD_SIZES = [64, 80, 96, 112, 128, 256
32
33 BLOCK_SIZES = [16, 32]
34 USE_ALIBI = [False, True]
35 -KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
35 +KV_CACHE_DTYPE = ["auto", "fp8"]
36 SEEDS = [0]
37 CUDA_DEVICES = [
38 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
@@ -172,6 +172,9 @@ def test_paged_attention(
172 device)
173 key_cache, value_cache = key_caches[0], value_caches[0]
174
175 + # Using default kv_scale
176 + kv_scale = 1.0
177 +
178 # Call the paged attention kernel.
179 output = torch.empty_like(query)
180 if version == "v1":
@@ -188,6 +191,7 @@ def test_paged_attention(
191 max_context_len,
192 alibi_slopes,
193 kv_cache_dtype,
194 + kv_scale,
195 )
196 elif version == "v2":
197 num_partitions = ((max_context_len + PARTITION_SIZE - 1) //
@@ -219,12 +223,13 @@ def test_paged_attention(
223 max_context_len,
224 alibi_slopes,
225 kv_cache_dtype,
226 + kv_scale,
227 )
228 else:
229 raise AssertionError(f"Unknown version: {version}")
230
231 # Run the reference implementation.
227 - if kv_cache_dtype == "fp8_e5m2":
232 + if kv_cache_dtype == "fp8":
233 # Convert cache data back to dtype.
234 x = 16 // torch.tensor([], dtype=dtype).element_size()
235 key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
@@ -232,14 +237,14 @@ def test_paged_attention(
237 dequantized_key_cache = torch.empty(size=key_cache_shape,
238 dtype=dtype,
239 device=device)
235 - cache_ops.convert_fp8_e5m2(key_cache, dequantized_key_cache)
240 + cache_ops.convert_fp8(key_cache, dequantized_key_cache)
241 key_cache = dequantized_key_cache
242
243 value_cache_shape = value_cache.shape
244 dequantized_value_cache = torch.empty(size=value_cache_shape,
245 dtype=dtype,
246 device=device)
242 - cache_ops.convert_fp8_e5m2(value_cache, dequantized_value_cache)
247 + cache_ops.convert_fp8(value_cache, dequantized_value_cache)
248 value_cache = dequantized_value_cache
249
250 ref_output = torch.empty_like(query)
@@ -263,7 +268,8 @@ def test_paged_attention(
268
269 # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
270 # so we use a relaxed tolerance for the test.
266 - if kv_cache_dtype == "fp8_e5m2":
271 + atol, rtol = 1e-3, 1e-5
272 + if kv_cache_dtype == "fp8":
273 atol, rtol = 1e-2, 1e-5
274 assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
275
tests/kernels/test_cache.py
+86 -12
@@ -5,6 +5,7 @@ import pytest
5 import torch
6
7 from vllm._C import cache_ops
8 +from vllm.utils import is_hip
9
10 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
11 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -23,7 +24,7 @@ SEEDS = [0]
24 CUDA_DEVICES = [
25 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
26 ]
26 -KV_CACHE_DTYPE = ["auto", "fp8_e5m2"]
27 +KV_CACHE_DTYPE = ["auto", "fp8"]
28
29
30 @pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@@ -105,6 +106,7 @@ def test_copy_blocks(
106 @pytest.mark.parametrize("dtype", DTYPES)
107 @pytest.mark.parametrize("seed", SEEDS)
108 @pytest.mark.parametrize("device", CUDA_DEVICES)
109 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
110 @torch.inference_mode()
111 def test_reshape_and_cache(
112 kv_cache_factory,
@@ -116,7 +118,10 @@ def test_reshape_and_cache(
118 dtype: torch.dtype,
119 seed: int,
120 device: str,
121 + kv_cache_dtype: str,
122 ) -> None:
123 + if not is_hip() and kv_cache_dtype == "fp8":
124 + pytest.skip() # This test is not tuned for e5m2 cuda precision
125 random.seed(seed)
126 torch.random.manual_seed(seed)
127 if torch.cuda.is_available():
@@ -132,17 +137,33 @@ def test_reshape_and_cache(
137
138 # Create the KV caches.
139 key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
135 - num_heads, head_size, dtype,
136 - None, seed, device)
140 + num_heads, head_size,
141 + kv_cache_dtype, dtype, seed,
142 + device)
143 key_cache, value_cache = key_caches[0], value_caches[0]
144
145 # Clone the KV caches.
140 - cloned_key_cache = key_cache.clone()
141 - cloned_value_cache = value_cache.clone()
146 + if kv_cache_dtype == "fp8":
147 + cloned_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
148 + cache_ops.convert_fp8(key_cache, cloned_key_cache)
149 + cloned_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
150 + cache_ops.convert_fp8(value_cache, cloned_value_cache)
151 + else:
152 + cloned_key_cache = key_cache.clone()
153 + cloned_value_cache = value_cache.clone()
154 +
155 + # Using default kv_scale
156 + kv_scale = 1.0
157
158 # Call the reshape_and_cache kernel.
159 cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
145 - slot_mapping, "auto")
160 + slot_mapping, kv_cache_dtype, kv_scale)
161 +
162 + if kv_cache_dtype == "fp8":
163 + result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
164 + cache_ops.convert_fp8(key_cache, result_key_cache)
165 + result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
166 + cache_ops.convert_fp8(value_cache, result_value_cache)
167
168 # Run the reference implementation.
169 reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
@@ -156,8 +177,18 @@ def test_reshape_and_cache(
177 cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
178 cloned_value_cache[block_idx, :, :, block_offset] = value[i]
179
159 - assert torch.allclose(key_cache, cloned_key_cache)
160 - assert torch.allclose(value_cache, cloned_value_cache)
180 + if kv_cache_dtype == "fp8":
181 + assert torch.allclose(result_key_cache,
182 + cloned_key_cache,
183 + atol=0.001,
184 + rtol=0.1)
185 + assert torch.allclose(result_value_cache,
186 + cloned_value_cache,
187 + atol=0.001,
188 + rtol=0.1)
189 + else:
190 + assert torch.allclose(key_cache, cloned_key_cache)
191 + assert torch.allclose(value_cache, cloned_value_cache)
192
193
194 @pytest.mark.parametrize("direction", COPYING_DIRECTION)
@@ -169,6 +200,7 @@ def test_reshape_and_cache(
200 @pytest.mark.parametrize("dtype", DTYPES)
201 @pytest.mark.parametrize("seed", SEEDS)
202 @pytest.mark.parametrize("device", CUDA_DEVICES)
203 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
204 @torch.inference_mode()
205 def test_swap_blocks(
206 kv_cache_factory,
@@ -181,7 +213,12 @@ def test_swap_blocks(
213 dtype: torch.dtype,
214 seed: int,
215 device: str,
216 + kv_cache_dtype: str,
217 ) -> None:
218 + if kv_cache_dtype == "fp8" and "cpu" in direction:
219 + pytest.skip()
220 + if not is_hip() and kv_cache_dtype == "fp8":
221 + pytest.skip() # This test is not tuned for e5m2 cuda precision
222 random.seed(seed)
223 torch.random.manual_seed(seed)
224 if torch.cuda.is_available():
@@ -202,13 +239,13 @@ def test_swap_blocks(
239
240 # Create the KV caches on the first device.
241 src_key_caches, src_value_caches = kv_cache_factory(
205 - num_blocks, block_size, 1, num_heads, head_size, dtype, None, seed,
206 - src_device)
242 + num_blocks, block_size, 1, num_heads, head_size, kv_cache_dtype, dtype,
243 + seed, src_device)
244
245 # Create the KV caches on the second device.
246 dist_key_caches, dist_value_caches = kv_cache_factory(
210 - num_blocks, block_size, 1, num_heads, head_size, dtype, None, seed,
211 - dst_device)
247 + num_blocks, block_size, 1, num_heads, head_size, kv_cache_dtype, dtype,
248 + seed, dst_device)
249
250 src_key_caches_clone = src_key_caches[0].clone()
251 src_value_caches_clone = src_value_caches[0].clone()
@@ -223,3 +260,40 @@ def test_swap_blocks(
260 dist_key_caches[0][dst].cpu())
261 assert torch.allclose(src_value_caches_clone[src].cpu(),
262 dist_value_caches[0][dst].cpu())
263 +
264 +
265 +@pytest.mark.skipif(not is_hip(), reason="FP8 conversion test requires e4m3")
266 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
267 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
268 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
269 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
270 +@pytest.mark.parametrize("dtype", DTYPES)
271 +@pytest.mark.parametrize("seed", SEEDS)
272 +@pytest.mark.parametrize("device", CUDA_DEVICES)
273 +@torch.inference_mode()
274 +def test_fp8_conversion(
275 + num_heads: int,
276 + head_size: int,
277 + block_size: int,
278 + num_blocks: int,
279 + dtype: torch.dtype,
280 + seed: int,
281 + device: str,
282 +) -> None:
283 + random.seed(seed)
284 + torch.random.manual_seed(seed)
285 + torch.cuda.manual_seed(seed)
286 +
287 + low = -224.0
288 + high = 224.0
289 + shape = (num_blocks, num_heads, head_size, block_size)
290 + cache = torch.empty(shape, dtype=dtype, device=device)
291 + cache.uniform_(low, high)
292 +
293 + cache_fp8 = torch.empty_like(cache, dtype=torch.uint8)
294 + cache_ops.convert_fp8(cache, cache_fp8)
295 +
296 + converted_cache = torch.empty_like(cache)
297 + cache_ops.convert_fp8(cache_fp8, converted_cache)
298 +
299 + assert torch.allclose(cache, converted_cache, atol=0.001, rtol=0.1)
vllm/attention/backends/abstract.py
+1
@@ -81,5 +81,6 @@ class AttentionImpl(ABC):
81 value: torch.Tensor,
82 kv_cache: torch.Tensor,
83 attn_metadata: AttentionMetadata,
84 + kv_scale: float,
85 ) -> torch.Tensor:
86 raise NotImplementedError
vllm/attention/backends/flash_attn.py
+7 -1
@@ -156,6 +156,7 @@ class FlashAttentionImpl(AttentionImpl):
156 value: torch.Tensor,
157 kv_cache: torch.Tensor,
158 attn_metadata: FlashAttentionMetadata,
159 + kv_scale: float,
160 ) -> torch.Tensor:
161 """Forward pass with FlashAttention and PagedAttention.
162
@@ -184,7 +185,8 @@ class FlashAttentionImpl(AttentionImpl):
185 PagedAttention.write_to_paged_cache(key, value, key_cache,
186 value_cache,
187 attn_metadata.slot_mapping,
187 - attn_metadata.kv_cache_dtype)
188 + attn_metadata.kv_cache_dtype,
189 + kv_scale)
190
191 if attn_metadata.is_prompt:
192 # Prompt run.
@@ -207,6 +209,9 @@ class FlashAttentionImpl(AttentionImpl):
209 )
210 else:
211 # prefix-enabled attention
212 + # TODO(Hai) this triton kernel has regression issue (broke) to
213 + # deal with different data types between KV and FP8 KV cache,
214 + # to be addressed separately.
215 output = PagedAttention.forward_prefix(
216 query,
217 key,
@@ -233,6 +238,7 @@ class FlashAttentionImpl(AttentionImpl):
238 self.num_kv_heads,
239 self.scale,
240 self.alibi_slopes,
241 + kv_scale,
242 )
243
244 # Reshape the output tensor.
vllm/attention/backends/xformers.py
+7 -1
@@ -178,6 +178,7 @@ class XFormersImpl(AttentionImpl):
178 value: torch.Tensor,
179 kv_cache: Optional[torch.Tensor],
180 attn_metadata: XFormersMetadata,
181 + kv_scale: float,
182 ) -> torch.Tensor:
183 """Forward pass with xFormers and PagedAttention.
184
@@ -205,7 +206,8 @@ class XFormersImpl(AttentionImpl):
206 PagedAttention.write_to_paged_cache(key, value, key_cache,
207 value_cache,
208 attn_metadata.slot_mapping,
208 - attn_metadata.kv_cache_dtype)
209 + attn_metadata.kv_cache_dtype,
210 + kv_scale)
211
212 if attn_metadata.is_prompt:
213 # Prompt run.
@@ -259,6 +261,9 @@ class XFormersImpl(AttentionImpl):
261 query, key, value, attn_metadata)
262 else:
263 # prefix-enabled attention
264 + # TODO(Hai) this triton kernel has regression issue (broke) to
265 + # deal with different data types between KV and FP8 KV cache,
266 + # to be addressed separately.
267 output = PagedAttention.forward_prefix(
268 query,
269 key,
@@ -285,6 +290,7 @@ class XFormersImpl(AttentionImpl):
290 self.num_kv_heads,
291 self.scale,
292 self.alibi_slopes,
293 + kv_scale,
294 )
295
296 # Reshape the output tensor.
vllm/attention/layer.py
+3 -1
@@ -42,5 +42,7 @@ class Attention(nn.Module):
42 value: torch.Tensor,
43 kv_cache: Optional[torch.Tensor],
44 attn_metadata: AttentionMetadata,
45 + kv_scale: float = 1.0,
46 ) -> torch.Tensor:
46 - return self.impl.forward(query, key, value, kv_cache, attn_metadata)
47 + return self.impl.forward(query, key, value, kv_cache, attn_metadata,
48 + kv_scale)
vllm/attention/ops/paged_attn.py
+5
@@ -73,6 +73,7 @@ class PagedAttention:
73 value_cache: torch.Tensor,
74 slot_mapping: torch.Tensor,
75 kv_cache_dtype: str,
76 + kv_scale: float,
77 ) -> None:
78 cache_ops.reshape_and_cache(
79 key,
@@ -81,6 +82,7 @@ class PagedAttention:
82 value_cache,
83 slot_mapping.flatten(),
84 kv_cache_dtype,
85 + kv_scale,
86 )
87
88 @staticmethod
@@ -95,6 +97,7 @@ class PagedAttention:
97 num_kv_heads: int,
98 scale: float,
99 alibi_slopes: Optional[torch.Tensor],
100 + kv_scale,
101 ) -> torch.Tensor:
102 output = torch.empty_like(query)
103
@@ -126,6 +129,7 @@ class PagedAttention:
129 max_context_len,
130 alibi_slopes,
131 kv_cache_dtype,
132 + kv_scale,
133 )
134 else:
135 # Run PagedAttention V2.
@@ -157,6 +161,7 @@ class PagedAttention:
161 max_context_len,
162 alibi_slopes,
163 kv_cache_dtype,
164 + kv_scale,
165 )
166 return output
167
vllm/config.py
+20 -14
@@ -60,6 +60,11 @@ class ModelConfig:
60 output). If None, will be derived from the model.
61 quantization: Quantization method that was used to quantize the model
62 weights. If None, we assume the model weights are not quantized.
63 + quantization_param_path: Path to JSON file containing scaling factors.
64 + Used to load KV cache scaling factors into the model when KV cache
65 + type is FP8_E4M3 on ROCm (AMD GPU). In the future these will also
66 + be used to load activation and weight scaling factors when the
67 + model dtype is FP8_E4M3 on ROCm.
68 enforce_eager: Whether to enforce eager execution. If True, we will
69 disable CUDA graph and always execute the model in eager mode.
70 If False, we will use CUDA graph and eager execution in hybrid.
@@ -83,6 +88,7 @@ class ModelConfig:
88 tokenizer_revision: Optional[str] = None,
89 max_model_len: Optional[int] = None,
90 quantization: Optional[str] = None,
91 + quantization_param_path: Optional[str] = None,
92 enforce_eager: bool = False,
93 max_context_len_to_capture: Optional[int] = None,
94 max_logprobs: int = 5,
@@ -98,6 +104,7 @@ class ModelConfig:
104 self.code_revision = code_revision
105 self.tokenizer_revision = tokenizer_revision
106 self.quantization = quantization
107 + self.quantization_param_path = quantization_param_path
108 self.enforce_eager = enforce_eager
109 self.max_context_len_to_capture = max_context_len_to_capture
110 self.max_logprobs = max_logprobs
@@ -369,21 +376,20 @@ class CacheConfig:
376 def _verify_cache_dtype(self) -> None:
377 if self.cache_dtype == "auto":
378 pass
372 - elif self.cache_dtype == "fp8_e5m2":
373 - if is_hip():
374 - raise NotImplementedError(
375 - "FP8_E5M2 KV Cache on AMD GPU has not been supported yet.")
376 - nvcc_cuda_version = get_nvcc_cuda_version()
377 - if nvcc_cuda_version and nvcc_cuda_version < Version("11.8"):
378 - raise ValueError(
379 - "FP8 is not supported when cuda version is lower than 11.8."
380 - )
379 + elif self.cache_dtype == "fp8":
380 + if not is_hip():
381 + nvcc_cuda_version = get_nvcc_cuda_version()
382 + if nvcc_cuda_version < Version("11.8"):
383 + raise ValueError(
384 + "FP8 is not supported when cuda version is"
385 + "lower than 11.8.")
386 logger.info(
382 - "Using fp8_e5m2 data type to store kv cache. It reduces "
383 - "the GPU memory footprint and boosts the performance. "
384 - "But it may cause slight accuracy drop. "
385 - "Currently we only support fp8 without scaling factors and "
386 - "make e5m2 as a default format.")
387 + "Using fp8 data type to store kv cache. It reduces the GPU "
388 + "memory footprint and boosts the performance. "
389 + "But it may cause slight accuracy drop without scaling "
390 + "factors. FP8_E5M2 (without scaling) is only supported on "
391 + "cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 "
392 + "is instead supported for common inference criteria.")
393 else:
394 raise ValueError(f"Unknown kv cache dtype: {self.cache_dtype}")
395
vllm/engine/arg_utils.py
+18 -5
@@ -21,6 +21,7 @@ class EngineArgs:
21 load_format: str = 'auto'
22 dtype: str = 'auto'
23 kv_cache_dtype: str = 'auto'
24 + quantization_param_path: Optional[str] = None
25 seed: int = 0
26 max_model_len: Optional[int] = None
27 worker_use_ray: bool = False
@@ -159,11 +160,23 @@ class EngineArgs:
160 parser.add_argument(
161 '--kv-cache-dtype',
162 type=str,
162 - choices=['auto', 'fp8_e5m2'],
163 + choices=['auto', 'fp8'],
164 default=EngineArgs.kv_cache_dtype,
165 help='Data type for kv cache storage. If "auto", will use model '
165 - 'data type. Note FP8 is not supported when cuda version is '
166 - 'lower than 11.8.')
166 + 'data type. FP8_E5M2 (without scaling) is only supported on cuda '
167 + 'version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
168 + 'supported for common inference criteria. ')
169 + parser.add_argument(
170 + '--quantization-param-path',
171 + type=str,
172 + default=None,
173 + help='Path to the JSON file containing the KV cache '
174 + 'scaling factors. This should generally be supplied, when '
175 + 'KV cache dtype is FP8. Otherwise, KV cache scaling factors '
176 + 'default to 1.0, which may cause accuracy issues. '
177 + 'FP8_E5M2 (without scaling) is only supported on cuda version'
178 + 'greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is instead '
179 + 'supported for common inference criteria. ')
180 parser.add_argument('--max-model-len',
181 type=int,
182 default=EngineArgs.max_model_len,
@@ -408,8 +421,8 @@ class EngineArgs:
421 self.trust_remote_code, self.download_dir, self.load_format,
422 self.dtype, self.seed, self.revision, self.code_revision,
423 self.tokenizer_revision, self.max_model_len, self.quantization,
411 - self.enforce_eager, self.max_context_len_to_capture,
412 - self.max_logprobs)
424 + self.quantization_param_path, self.enforce_eager,
425 + self.max_context_len_to_capture, self.max_logprobs)
426 cache_config = CacheConfig(self.block_size,
427 self.gpu_memory_utilization,
428 self.swap_space, self.kv_cache_dtype,
vllm/engine/llm_engine.py
+1
@@ -97,6 +97,7 @@ class LLMEngine:
97 f"quantization={model_config.quantization}, "
98 f"enforce_eager={model_config.enforce_eager}, "
99 f"kv_cache_dtype={cache_config.cache_dtype}, "
100 + f"quantization_param_path={model_config.quantization_param_path}, "
101 f"device_config={device_config.device}, "
102 f"seed={model_config.seed})")
103 # TODO(woosuk): Print more configs in debug mode.
vllm/model_executor/layers/quantization/schema.py new
+84
@@ -0,0 +1,84 @@
1 +"""
2 +This file contains the Pydantic schemas for various quantization-related
3 +parameters. When a relevant quantization technique is specified, these
4 +parameters are loaded in the form of a JSON alongside the model weights
5 +and augment the model with additional information needed for use of that
6 +technique. The format of this JSON should be specified by one or more
7 +schemas contained here.
8 +
9 +For example, when the KV cache is quantized to FP8-E4M3 (currently only
10 +possible on ROCm), the model can be optionally augmented with KV cache
11 +scaling factors.
12 +"""
13 +
14 +from typing import Dict, Optional
15 +
16 +from pydantic import BaseModel, ConfigDict, ValidationInfo, model_validator
17 +
18 +
19 +class KVCacheQuantSchema(BaseModel):
20 + dtype: str
21 + # Each key is a TP rank. Each value is a dictionary mapping a TP rank's
22 + # layer indices to their per-tensor KV cache scaling factor.
23 + # TODO: Consider pulling this and its validation methods out into its
24 + # own schema class (tricky as its members are variable)
25 + scaling_factor: Dict[int, Dict[int, float]]
26 +
27 + @model_validator(mode="after")
28 + def check_is_fp8(self) -> "KVCacheQuantSchema":
29 + assert self.dtype == "float8_e4m3fn", (
30 + "Loaded scaling factors intended for KV cache dtype = "
31 + f"{self.dtype} rather than float8_e4m3fn!")
32 + return self
33 +
34 + @model_validator(mode="after")
35 + def check_tp_ranks(self, info: ValidationInfo) -> "KVCacheQuantSchema":
36 + context = info.context
37 + if context:
38 + tp_size = context["tp_size"]
39 + num_hidden_layers = context["num_hidden_layers"]
40 + assert len(self.scaling_factor) == tp_size, (
41 + f"Loaded dictionary has TP size {len(self.scaling_factor)} "
42 + f"but LLM engine is currently running with TP size {tp_size}.")
43 + for tp_rank, layer_maps in self.scaling_factor.items():
44 + assert len(layer_maps) == num_hidden_layers, (
45 + f"KV cache scales map for TP rank {tp_rank} is malformed. "
46 + f"Expected {num_hidden_layers} layers, got "
47 + f"{len(layer_maps)}.")
48 + for i in range(tp_size):
49 + assert i in self.scaling_factor, (
50 + f"KV cache scales map for TP rank {i} not found.")
51 + return self
52 +
53 + @model_validator(mode="after")
54 + def check_current_rank(self, info: ValidationInfo) -> "KVCacheQuantSchema":
55 + context = info.context
56 + if context:
57 + tp_rank = context["tp_rank"]
58 + num_hidden_layers = context["num_hidden_layers"]
59 + layer_scales_map = self.scaling_factor[tp_rank]
60 + for i in range(num_hidden_layers):
61 + assert i in layer_scales_map, (
62 + f"Could not find KV cache scales for layer {i} in "
63 + f"TP rank {tp_rank}.")
64 + return self
65 +
66 +
67 +class QuantParamSchema(BaseModel):
68 + # TODO: Generalize and extend with more fields
69 + # (e.g. weights/activations params) once functionality is enabled
70 + model_config = ConfigDict(protected_namespaces=())
71 + model_type: Optional[str]
72 + kv_cache: KVCacheQuantSchema
73 +
74 + @model_validator(mode="after")
75 + def check_model_type(self, info: ValidationInfo) -> "QuantParamSchema":
76 + context = info.context
77 + if context:
78 + model_type = context.get("model_type", None)
79 + if model_type is not None:
80 + assert model_type == self.model_type, (
81 + f"Model type is {model_type} but loaded "
82 + f"scaling factors belonging to different "
83 + f"model type {self.model_type}!")
84 + return self
vllm/model_executor/models/llama.py
+39 -3
@@ -41,11 +41,13 @@ from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.parallel_utils.parallel_state import (
44 - get_tensor_model_parallel_world_size)
44 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.model_executor.weight_utils import (default_weight_loader,
47 - hf_model_weights_iterator)
47 + hf_model_weights_iterator,
48 + kv_cache_scales_loader)
49 from vllm.sequence import SamplerOutput
50 +from vllm.utils import is_hip
51
52
53 class LlamaMLP(nn.Module):
@@ -115,6 +117,15 @@ class LlamaAttention(nn.Module):
117 self.rope_theta = rope_theta
118 self.max_position_embeddings = max_position_embeddings
119
120 + # This will be overwritten by model initialization if we are using it.
121 + # N.B. currently we only support per tensor scalar scaling factors
122 + # & only applicable to ROCm (AMD GPU).
123 + # The scaling factor convention we are assuming is
124 + # quantized_value * scaling_factor ~= true_value
125 + # which is consistent with the practice of setting
126 + # scaling_factor = tensor_amax / FPtype_max
127 + self.kv_scale = 1.0
128 +
129 self.qkv_proj = QKVParallelLinear(
130 hidden_size,
131 self.head_dim,
@@ -153,7 +164,8 @@ class LlamaAttention(nn.Module):
164 qkv, _ = self.qkv_proj(hidden_states)
165 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
166 q, k = self.rotary_emb(positions, q, k)
156 - attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
167 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata,
168 + self.kv_scale)
169 output, _ = self.o_proj(attn_output)
170 return output
171
@@ -402,3 +414,27 @@ class LlamaForCausalLM(nn.Module):
414 weight_loader = getattr(param, "weight_loader",
415 default_weight_loader)
416 weight_loader(param, loaded_weight)
417 +
418 + # If this function is called, it should always initialize KV cache scale
419 + # factors (or else raise an exception). Thus, handled exceptions should
420 + # make sure to leave KV cache scale factors in a known good (dummy) state
421 + def load_kv_cache_scales(self, quantization_param_path: str) -> None:
422 + tp_size = get_tensor_model_parallel_world_size()
423 + tp_rank = get_tensor_model_parallel_rank()
424 + for layer_idx, scaling_factor in kv_cache_scales_loader(
425 + quantization_param_path, tp_rank, tp_size,
426 + self.config.num_hidden_layers,
427 + self.config.__class__.model_type):
428 + layer_self_attn = self.model.layers[layer_idx].self_attn
429 +
430 + if is_hip():
431 + # The scaling factor convention we are assuming is
432 + # quantized_value * scaling_factor ~= true_value
433 + # which is consistent with the practice of setting
434 + # scaling_factor = tensor_amax / FPtype_max
435 + scaling_factor *= 2
436 + if hasattr(layer_self_attn, "kv_scale"):
437 + layer_self_attn.kv_scale = scaling_factor
438 + else:
439 + raise RuntimeError("Self attention has no KV cache scaling "
440 + "factor attribute!")
vllm/model_executor/weight_utils.py
+42 -1
@@ -5,7 +5,7 @@ import hashlib
5 import json
6 import os
7 from collections import defaultdict
8 -from typing import Any, Iterator, List, Optional, Tuple
8 +from typing import Any, Iterable, Iterator, List, Optional, Tuple
9
10 import filelock
11 import numpy as np
@@ -18,6 +18,7 @@ from vllm.config import ModelConfig
18 from vllm.logger import init_logger
19 from vllm.model_executor.layers.quantization import (QuantizationConfig,
20 get_quantization_config)
21 +from vllm.model_executor.layers.quantization.schema import QuantParamSchema
22
23 logger = init_logger(__name__)
24
@@ -275,6 +276,46 @@ def hf_model_weights_iterator(
276 torch.cuda.empty_cache()
277
278
279 +def kv_cache_scales_loader(
280 + filename: str, tp_rank: int, tp_size: int, num_hidden_layers: int,
281 + model_type: Optional[str]) -> Iterable[Tuple[int, float]]:
282 + """
283 + A simple utility to read in KV cache scaling factors that have been
284 + previously serialized to disk. Used by the model to populate the appropriate
285 + KV cache scaling factors. The serialization should represent a dictionary
286 + whose keys are the TP ranks and values are another dictionary mapping layers
287 + to their KV cache scaling factors.
288 + Keep this function in sync with the output of examples/fp8/extract_scales.py
289 + """
290 + try:
291 + with open(filename) as f:
292 + context = {
293 + "model_type": model_type,
294 + "num_hidden_layers": num_hidden_layers,
295 + "tp_rank": tp_rank,
296 + "tp_size": tp_size,
297 + }
298 + schema_dct = json.load(f)
299 + schema = QuantParamSchema.model_validate(schema_dct,
300 + context=context)
301 + layer_scales_map = schema.kv_cache.scaling_factor[tp_rank]
302 + return layer_scales_map.items()
303 +
304 + except FileNotFoundError:
305 + logger.error(f"File or directory '{filename}' not found.")
306 + except json.JSONDecodeError:
307 + logger.error(f"Error decoding JSON in file '{filename}'.")
308 + except Exception as e:
309 + logger.error(f"An error occurred while reading '{filename}': {e}")
310 + # This section is reached if and only if any of the excepts are hit
311 + # Return an empty iterable (list) => no KV cache scales are loaded
312 + # which ultimately defaults to 1.0 scales
313 + logger.warning("Defaulting to KV cache scaling factors = 1.0 "
314 + f"for all layers in TP rank {tp_rank} "
315 + "as an error occurred during loading.")
316 + return []
317 +
318 +
319 def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
320 """convert PySafeSlice object from safetensors to torch.Tensor
321
vllm/utils.py
+10 -10
@@ -25,7 +25,7 @@ STR_DTYPE_TO_TORCH_DTYPE = {
25 "half": torch.half,
26 "bfloat16": torch.bfloat16,
27 "float": torch.float,
28 - "fp8_e5m2": torch.uint8,
28 + "fp8": torch.uint8,
29 }
30
31
@@ -266,7 +266,7 @@ def get_nvcc_cuda_version() -> Optional[Version]:
266 return nvcc_cuda_version
267
268
269 -def _generate_random_fp8_e5m2(
269 +def _generate_random_fp8(
270 tensor: torch.tensor,
271 low: float,
272 high: float,
@@ -282,7 +282,7 @@ def _generate_random_fp8_e5m2(
282 from vllm._C import cache_ops
283 tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
284 tensor_tmp.uniform_(low, high)
285 - cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)
285 + cache_ops.convert_fp8(tensor_tmp, tensor)
286 del tensor_tmp
287
288
@@ -311,7 +311,7 @@ def create_kv_caches_with_random(
311 raise ValueError(f"Invalid model dtype: {model_dtype}")
312 elif cache_dtype in ["half", "bfloat16", "float"]:
313 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
314 - elif cache_dtype == "fp8_e5m2":
314 + elif cache_dtype == "fp8":
315 torch_dtype = torch.uint8
316 else:
317 raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
@@ -328,10 +328,10 @@ def create_kv_caches_with_random(
328 key_cache = torch.empty(size=key_cache_shape,
329 dtype=torch_dtype,
330 device=device)
331 - if cache_dtype == 'fp8_e5m2':
332 - _generate_random_fp8_e5m2(key_cache, -scale, scale)
333 - elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
331 + if cache_dtype in ["auto", "half", "bfloat16", "float"]:
332 key_cache.uniform_(-scale, scale)
333 + elif cache_dtype == 'fp8':
334 + _generate_random_fp8(key_cache, -scale, scale)
335 else:
336 raise ValueError(
337 f"Does not support key cache of type {cache_dtype}")
@@ -343,10 +343,10 @@ def create_kv_caches_with_random(
343 value_cache = torch.empty(size=value_cache_shape,
344 dtype=torch_dtype,
345 device=device)
346 - if cache_dtype == 'fp8_e5m2':
347 - _generate_random_fp8_e5m2(value_cache, -scale, scale)
348 - elif torch_dtype in [torch.half, torch.bfloat16, torch.float]:
346 + if cache_dtype in ["auto", "half", "bfloat16", "float"]:
347 value_cache.uniform_(-scale, scale)
348 + elif cache_dtype == 'fp8':
349 + _generate_random_fp8(value_cache, -scale, scale)
350 else:
351 raise ValueError(
352 f"Does not support value cache of type {cache_dtype}")
vllm/worker/model_runner.py
+21 -1
@@ -23,7 +23,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
23 from vllm.sampling_params import SamplingParams, SamplingType
24 from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
25 SequenceGroupMetadata)
26 -from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d,
26 +from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d, is_hip,
27 is_pin_memory_available, make_tensor_with_pad,
28 maybe_expand_dim)
29
@@ -120,6 +120,26 @@ class ModelRunner:
120 self.model.embedding_padding_modules)
121 self.model = self.lora_manager.create_lora_manager(self.model)
122
123 + if self.kv_cache_dtype == "fp8" and is_hip():
124 + # Currently scaled KV cache is only enabled on ROCm
125 + if self.model_config.quantization_param_path is not None:
126 + if callable(getattr(self.model, "load_kv_cache_scales", None)):
127 + self.model.load_kv_cache_scales(
128 + self.model_config.quantization_param_path)
129 + else:
130 + raise RuntimeError("Using FP8 KV cache and scaling "
131 + "factors provided but model "
132 + f"{self.model.__class__} does not "
133 + "support loading scaling factors.")
134 + else:
135 + logger.warn("Using FP8 KV cache but no scaling factors "
136 + "provided. Defaulting to scaling factors of 1.0. "
137 + "This may lead to less accurate results!")
138 + elif self.model_config.quantization_param_path is not None:
139 + logger.warn("KV cache scaling factors provided, "
140 + "but the KV cache data type is not FP8. "
141 + "KV cache scaling factors will not be used.")
142 +
143 def set_block_size(self, block_size: int) -> None:
144 self.block_size = block_size
145