@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b00b33d77e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model][Quantization] HQQ support through Marlin kernel expansion (#9766)

Signed-off-by: ElizaWszola <eliza@neuralmagic.com>

ElizaWszola committed Nov 19, 2024 at 22:31 UTC b00b33d77e33c5516e73de663539dff96e8b61a4
11 files changed +632 -89
benchmarks/kernels/benchmark_machete.py
+2 -1
@@ -210,7 +210,8 @@ def marlin_create_bench_fn(bt: BenchmarkTensors) -> Callable:
210 size_m=bt.a.shape[0],
211 size_n=bt.w_ref.shape[1],
212 size_k=bt.w_ref.shape[0],
213 - is_k_full=True)
213 + is_k_full=True,
214 + is_zp_float=False)
215 else:
216 assert bt.a.dtype == torch.int8
217 assert bt.wtype == scalar_types.uint4b8
benchmarks/kernels/benchmark_marlin.py
+2 -2
@@ -131,7 +131,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
131 results.append(
132 benchmark.Timer(
133 stmt=
134 - "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
134 + "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, False, False)", # noqa: E501
135 globals=globals,
136 label=label,
137 sub_label=sub_label,
@@ -141,7 +141,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
141 results.append(
142 benchmark.Timer(
143 stmt=
144 - "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
144 + "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, True, False)", # noqa: E501
145 globals=globals,
146 label=label,
147 sub_label=sub_label,
csrc/quantization/gptq_marlin/gptq_marlin.cu
+200 -77
@@ -54,9 +54,10 @@ template <typename scalar_t, // compute dtype, half or nv_float16
54 const int thread_k_blocks, // same for k dimension (reduction)
55 const int stages, // number of stages for the async global->shared
56 // fetch pipeline
57 - const bool has_act_order, // whether act_order is enabled
58 - const int group_blocks = -1 // number of consecutive 16x16 blocks
59 - // with a separate quantization scale
57 + const bool has_act_order, // whether act_order is enabled
58 + const int group_blocks = -1, // number of consecutive 16x16 blocks
59 + // with a separate quantization scale
60 + const bool is_zp_float // is zero point of float16 type?
61 >
62 __global__ void Marlin(
63 const int4* __restrict__ A, // fp16 input matrix of shape mxk
@@ -82,7 +83,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
83 torch::Tensor& workspace,
84 vllm::ScalarTypeId const b_q_type_id,
85 int64_t size_m, int64_t size_n, int64_t size_k,
85 - bool is_k_full, bool has_zp) {
86 + bool is_k_full, bool has_zp, bool is_zp_float) {
87 TORCH_CHECK_NOT_IMPLEMENTED(false,
88 "marlin_gemm(..) requires CUDA_ARCH >= 8.0");
89 return torch::empty({1, 1});
@@ -516,10 +517,11 @@ template <typename scalar_t, // compute dtype, half or nv_float16
517 const int thread_k_blocks, // same for k dimension (reduction)
518 const int stages, // number of stages for the async global->shared
519 // fetch pipeline
519 - const bool has_act_order, // whether act_order is enabled
520 - const bool has_zp, // whether zero-points are enabled
521 - const int group_blocks = -1 // number of consecutive 16x16 blocks
522 - // with a separate quantization scale
520 + const bool has_act_order, // whether act_order is enabled
521 + const bool has_zp, // whether zero-points are enabled
522 + const int group_blocks = -1, // number of consecutive 16x16 blocks
523 + // with a separate quantization scale
524 + const bool is_zp_float // is zero point of float16 type?
525 >
526 __global__ void Marlin(
527 const int4* __restrict__ A, // fp16 input matrix of shape mxk
@@ -692,8 +694,10 @@ __global__ void Marlin(
694 int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
695
696 // Zero-points sizes/strides
695 - int zp_gl_stride = (prob_n / pack_factor) / 4;
696 - constexpr int zp_sh_stride = ((16 * thread_n_blocks) / pack_factor) / 4;
697 + int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4;
698 + constexpr int zp_sh_stride = is_zp_float
699 + ? 16 * thread_n_blocks / 8
700 + : ((16 * thread_n_blocks) / pack_factor) / 4;
701 constexpr int zp_tb_groups = s_tb_groups;
702 constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0;
703 int zp_gl_rd_delta = zp_gl_stride;
@@ -768,9 +772,16 @@ __global__ void Marlin(
772 constexpr int num_ints_per_thread = 8 / pack_factor;
773 int zp_sh_rd;
774 if constexpr (has_zp) {
771 - zp_sh_rd = num_ints_per_thread * num_col_threads *
772 - ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
773 - num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
775 + if constexpr (is_zp_float) {
776 + if constexpr (group_blocks != -1) {
777 + zp_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
778 + (threadIdx.x % 32) / 4;
779 + }
780 + } else {
781 + zp_sh_rd = num_ints_per_thread * num_col_threads *
782 + ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
783 + num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
784 + }
785 }
786
787 // Precompute which thread should not read memory in which iterations; this is
@@ -832,6 +843,7 @@ __global__ void Marlin(
843 FragS act_frag_s[2][4][4]; // For act-order
844 int frag_qzp[2][num_ints_per_thread]; // Zero-points
845 FragZP frag_zp; // Zero-points in fp16
846 + FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ
847
848 // Zero accumulators.
849 auto zero_accums = [&]() {
@@ -1126,7 +1138,7 @@ __global__ void Marlin(
1138 // has_zp implies AWQ, which doesn't have act_order,
1139 static_assert(!has_zp || group_blocks != 0);
1140
1129 - if constexpr (has_zp) {
1141 + if constexpr (has_zp && !is_zp_float) {
1142 int pipe = full_pipe % stages;
1143
1144 if constexpr (group_blocks == -1) {
@@ -1170,11 +1182,44 @@ __global__ void Marlin(
1182 }
1183 }
1184 }
1185 +
1186 + else if constexpr (has_zp && is_zp_float) {
1187 + int pipe = full_pipe % stages;
1188 +
1189 + if constexpr (group_blocks != -1) {
1190 + if constexpr (group_blocks >= thread_k_blocks) {
1191 + int4* sh_zp_stage =
1192 + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1193 + (pipe / (group_blocks / thread_k_blocks)));
1194 + reinterpret_cast<int4*>(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd];
1195 + } else {
1196 + int warp_id = threadIdx.x / 32;
1197 + int n_warps = thread_n_blocks / 4;
1198 +
1199 + int warp_row = warp_id / n_warps;
1200 +
1201 + int cur_k = warp_row * 16;
1202 + cur_k += k_iter_size * (k % b_sh_wr_iters);
1203 +
1204 + int k_blocks = cur_k / 16;
1205 + // Suppress bogus and persistent divide-by-zero warning
1206 + #pragma nv_diagnostic push
1207 + #pragma nv_diag_suppress divide_by_zero
1208 + int cur_group_id = k_blocks / group_blocks;
1209 + #pragma nv_diagnostic pop
1210 +
1211 + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1212 +
1213 + reinterpret_cast<int4*>(&frag_zpf[k % 2])[0] =
1214 + sh_zp_stage[zp_sh_rd + cur_group_id * zp_sh_stride];
1215 + }
1216 + }
1217 + }
1218 };
1219
1220 // Execute the actual tensor core matmul of a sub-tile.
1221 auto matmul = [&](int k) {
1177 - if constexpr (has_zp) {
1222 + if constexpr (has_zp && !is_zp_float) {
1223 FragB frag_zp_0;
1224 FragB frag_zp_1;
1225 int zp_quant_0, zp_quant_1;
@@ -1219,10 +1264,14 @@ __global__ void Marlin(
1264 frag_b1 = dequant<scalar_t, w_type_id>(b_quant_1);
1265
1266 // Apply zero-point to frag_b0
1222 - if constexpr (has_zp) {
1267 + if constexpr (has_zp && !is_zp_float) {
1268 sub_zp<scalar_t>(frag_b0, frag_zp[j], 0);
1269 }
1270
1271 + else if constexpr (has_zp && is_zp_float && group_blocks != -1) {
1272 + sub_zp<scalar_t>(frag_b0, frag_zpf[k % 2][j], 0);
1273 + }
1274 +
1275 // Apply scale to frag_b0
1276 if constexpr (has_act_order) {
1277 scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j],
@@ -1235,10 +1284,14 @@ __global__ void Marlin(
1284 }
1285
1286 // Apply zero-point to frag_b1
1238 - if constexpr (has_zp) {
1287 + if constexpr (has_zp && !is_zp_float) {
1288 sub_zp<scalar_t>(frag_b1, frag_zp[j], 1);
1289 }
1290
1291 + else if constexpr (has_zp && is_zp_float && group_blocks != -1) {
1292 + sub_zp<scalar_t>(frag_b1, frag_zpf[k % 2][j], 1);
1293 + }
1294 +
1295 // Apply scale to frag_b1
1296 if constexpr (has_act_order) {
1297 scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j],
@@ -1510,7 +1563,7 @@ __global__ void Marlin(
1563 fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1564 }
1565
1513 - if constexpr (has_zp && group_blocks == -1) {
1566 + if constexpr (has_zp && !is_zp_float && group_blocks == -1) {
1567 if (i == 0) {
1568 fetch_zp_to_shared();
1569 }
@@ -1697,23 +1750,27 @@ __global__ void Marlin(
1750 }
1751
1752 #define __CALL_IF(W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1700 - HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, NUM_THREADS) \
1753 + HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, NUM_THREADS, \
1754 + IS_ZP_FLOAT) \
1755 else if (q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && \
1756 thread_n_blocks == THREAD_N_BLOCKS && \
1757 thread_k_blocks == THREAD_K_BLOCKS && \
1758 has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1705 - group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1706 - cudaFuncSetAttribute( \
1707 - Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1708 - THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1709 - HAS_ZP, GROUP_BLOCKS>, \
1710 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1711 - Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1712 - THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1713 - HAS_ZP, GROUP_BLOCKS> \
1714 - <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1715 - A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, zp_ptr, g_idx_ptr, \
1716 - num_groups, prob_m, prob_n, prob_k, locks, use_fp32_reduce); \
1759 + group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS && \
1760 + is_zp_float == IS_ZP_FLOAT) { \
1761 + if constexpr (!IS_ZP_FLOAT || std::is_same<scalar_t, half>::value) { \
1762 + cudaFuncSetAttribute( \
1763 + Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1764 + THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, \
1765 + HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, IS_ZP_FLOAT>, \
1766 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1767 + Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1768 + THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1769 + HAS_ZP, GROUP_BLOCKS, IS_ZP_FLOAT> \
1770 + <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1771 + A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, zp_ptr, g_idx_ptr, \
1772 + num_groups, prob_m, prob_n, prob_k, locks, use_fp32_reduce); \
1773 + } \
1774 }
1775
1776 typedef struct {
@@ -1905,51 +1962,96 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1962 }
1963
1964 #define GPTQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1908 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1909 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1910 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1911 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1965 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1966 + false) \
1967 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1968 + false) \
1969 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1970 + false) \
1971 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1972 + false) \
1973 \
1913 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1914 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1915 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1916 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1974 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
1975 + false) \
1976 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
1977 + false) \
1978 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
1979 + false) \
1980 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
1981 + false) \
1982 \
1918 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1919 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1920 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1921 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1983 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
1984 + false) \
1985 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
1986 + false) \
1987 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
1988 + false) \
1989 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
1990 + false) \
1991 \
1923 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1924 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1925 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1926 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1992 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
1993 + false) \
1994 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
1995 + false) \
1996 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
1997 + false) \
1998 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
1999 + false) \
2000 \
1928 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1929 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1930 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1931 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
2001 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
2002 + false) \
2003 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
2004 + false) \
2005 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
2006 + false) \
2007 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
2008 + false)
2009
2010 #define AWQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1934 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1935 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1936 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1937 - __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
2011 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2012 + false) \
2013 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2014 + false) \
2015 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2016 + false) \
2017 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, \
2018 + false) \
2019 \
1939 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1940 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1941 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1942 - __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
2020 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2021 + false) \
2022 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2023 + false) \
2024 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2025 + false) \
2026 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, \
2027 + false) \
2028 \
1944 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1945 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1946 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1947 - __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
2029 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2030 + false) \
2031 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2032 + false) \
2033 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2034 + false) \
2035 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, \
2036 + false) \
2037 \
1949 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1950 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1951 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1952 - __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
2038 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2039 + false) \
2040 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2041 + false) \
2042 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2043 + false) \
2044 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, false)
2045 +
2046 + // We currently have 4-bit models only with group_blocks == 4
2047 + #define HQQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
2048 + __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2049 + true) \
2050 + __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2051 + true) \
2052 + __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2053 + true) \
2054 + __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, true)
2055
2056 template <typename scalar_t>
2057 void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
@@ -1958,7 +2060,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
2060 vllm::ScalarType const& q_type, bool has_act_order,
2061 bool is_k_full, bool has_zp, int num_groups, int group_size,
2062 int dev, cudaStream_t stream, int thread_k, int thread_n,
1961 - int sms, int max_par, bool use_fp32_reduce) {
2063 + int sms, int max_par, bool use_fp32_reduce, bool is_zp_float) {
2064 if (has_zp) {
2065 TORCH_CHECK(
2066 q_type == vllm::kU4 || q_type == vllm::kU8,
@@ -2111,6 +2213,11 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
2213 AWQ_CALL_IF(vllm::kU8, 8, 8, 256)
2214 AWQ_CALL_IF(vllm::kU8, 8, 4, 128)
2215 AWQ_CALL_IF(vllm::kU8, 4, 8, 128)
2216 +
2217 + HQQ_CALL_IF(vllm::kU4, 16, 4, 256)
2218 + HQQ_CALL_IF(vllm::kU4, 8, 8, 256)
2219 + HQQ_CALL_IF(vllm::kU4, 8, 4, 128)
2220 + HQQ_CALL_IF(vllm::kU4, 4, 8, 128)
2221 else {
2222 TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
2223 ", ", prob_k, "]", ", has_act_order = ", has_act_order,
@@ -2135,7 +2242,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2242 vllm::ScalarTypeId const& b_q_type_id,
2243 int64_t size_m, int64_t size_n, int64_t size_k,
2244 bool is_k_full, bool has_zp,
2138 - bool use_fp32_reduce) {
2245 + bool use_fp32_reduce, bool is_zp_float) {
2246 vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
2247 if (has_zp) {
2248 TORCH_CHECK(
@@ -2148,6 +2255,12 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2255 b_q_type.str());
2256 }
2257
2258 + if (has_zp && is_zp_float) {
2259 + TORCH_CHECK(a.scalar_type() == at::ScalarType::Half,
2260 + "Computation type must be float16 (half) when using float zero "
2261 + "points.");
2262 + }
2263 +
2264 int pack_factor = 32 / b_q_type.size_bits();
2265
2266 // Verify A
@@ -2257,12 +2370,22 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2370 if (has_zp) {
2371 int rank = b_zeros.sizes().size();
2372 TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2");
2260 - TORCH_CHECK(b_zeros.size(0) == num_groups,
2261 - "b_zeros dim 0 = ", b_zeros.size(0),
2262 - " is not num_groups = ", num_groups);
2263 - TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2264 - "b_zeros dim 1 = ", b_zeros.size(1),
2265 - " is not size_n / pack_factor = ", size_n / pack_factor);
2373 + if (is_zp_float) {
2374 + TORCH_CHECK(b_zeros.size(1) == size_n,
2375 + "b_zeros dim 1 = ", b_zeros.size(1),
2376 + " is not size_n = ", size_n);
2377 + TORCH_CHECK(num_groups == b_zeros.size(0),
2378 + "b_zeros dim 0 = ", b_zeros.size(0),
2379 + " is not num_groups = ", num_groups);
2380 + TORCH_CHECK(num_groups != -1, "num_groups must be != -1");
2381 + } else {
2382 + TORCH_CHECK(b_zeros.size(0) == num_groups,
2383 + "b_zeros dim 0 = ", b_zeros.size(0),
2384 + " is not num_groups = ", num_groups);
2385 + TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2386 + "b_zeros dim 1 = ", b_zeros.size(1),
2387 + " is not size_n / pack_factor = ", size_n / pack_factor);
2388 + }
2389 }
2390
2391 // Verify workspace size
@@ -2282,7 +2405,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2405 a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2406 workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2407 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2285 - thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2408 + thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce, is_zp_float);
2409 } else if (a.scalar_type() == at::ScalarType::BFloat16) {
2410 marlin::marlin_mm<nv_bfloat16>(
2411 a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
@@ -2291,7 +2414,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2414 perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2415 workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2416 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2294 - thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2417 + thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce, is_zp_float);
2418 } else {
2419 TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
2420 }
csrc/torch_bindings.cpp
+1 -1
@@ -244,7 +244,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
244 "Tensor b_zeros, Tensor g_idx, Tensor perm, Tensor workspace, "
245 "int b_q_type, "
246 "SymInt size_m, SymInt size_n, SymInt size_k, bool is_k_full, "
247 - "bool has_zp, bool use_fp32_reduce) -> Tensor");
247 + "bool has_zp, bool use_fp32_reduce, bool is_zp_float) -> Tensor");
248 // conditionally compiled so impl registration is in source file
249
250 // gptq_marlin repack from GPTQ.
tests/kernels/test_marlin_gemm.py
+87 -1
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test_qqq import
29 marlin_qqq_quantize)
30 from vllm.model_executor.layers.quantization.utils.quant_utils import (
31 awq_pack, gptq_pack, gptq_quantize_weights, quantize_weights, sort_weights)
32 +from vllm.scalar_type import scalar_types
33
34 ACT_ORDER_OPTS = [False, True]
35 K_FULL_OPTS = [False, True]
@@ -40,6 +41,8 @@ MARLIN_N_CHUNKS = [64, 256]
41 MARLIN_24_K_CHUNKS = [128]
42 MARLIN_24_N_CHUNKS = [512]
43
44 +HQQ_SUPPORTED_GROUP_SIZES = [64]
45 +
46 MNK_FACTORS = [
47 (1, 1, 1),
48 (1, 4, 8),
@@ -226,7 +229,7 @@ def test_gptq_marlin_gemm(
229 torch.ops._C.gptq_marlin_gemm,
230 (a_input, marlin_q_w, marlin_s, marlin_zp, g_idx, sort_indices,
231 workspace.scratch, quant_type.id, a_input.shape[0], b_weight.shape[1],
229 - a_input.shape[1], is_k_full, False, use_fp32_reduce),
232 + a_input.shape[1], is_k_full, False, use_fp32_reduce, False),
233 test_utils=DEFAULT_OPCHECK_TEST_UTILS)
234
235 output = ops.gptq_marlin_gemm(
@@ -244,6 +247,7 @@ def test_gptq_marlin_gemm(
247 is_k_full=is_k_full,
248 has_zp=False,
249 use_fp32_reduce=use_fp32_reduce,
250 + is_zp_float=False,
251 )
252 output_ref = torch.matmul(a_input, w_ref)
253
@@ -441,6 +445,7 @@ def test_awq_marlin_gemm(
445 is_k_full=is_k_full,
446 has_zp=has_zp,
447 use_fp32_reduce=use_fp32_reduce,
448 + is_zp_float=False,
449 )
450 output_ref = torch.matmul(a_input, w_ref)
451
@@ -451,6 +456,87 @@ def test_awq_marlin_gemm(
456 assert max_diff < 0.04
457
458
459 +@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
460 + reason="Marlin is not supported on this GPU type.")
461 +@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
462 +@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
463 +@pytest.mark.parametrize("group_size", HQQ_SUPPORTED_GROUP_SIZES)
464 +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
465 +@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
466 +def test_hqq_marlin_gemm(
467 + k_chunk,
468 + n_chunk,
469 + group_size,
470 + mnk_factors,
471 + use_fp32_reduce,
472 +):
473 + m_factor, n_factor, k_factor = mnk_factors
474 +
475 + size_m = m_factor
476 + size_k = k_chunk * k_factor
477 + size_n = n_chunk * n_factor
478 +
479 + quant_type = scalar_types.uint4
480 +
481 + a_input = rand_data((size_m, size_k))
482 + dev = a_input.device
483 +
484 + b_weight = torch.randint(0,
485 + 10, (size_n, size_k),
486 + dtype=torch.uint8,
487 + device=dev)
488 + scale = rand_data((size_n, size_k // group_size))
489 + zero = rand_data((size_n, size_k // group_size))
490 +
491 + gptq_w_q = gptq_pack(b_weight.transpose(1, 0), 4, size_k, size_n)
492 +
493 + sort_indices = torch.empty(0, dtype=torch.int, device=dev)
494 + marlin_w_q = ops.gptq_marlin_repack(gptq_w_q, sort_indices, size_k, size_n,
495 + 4).to(dev)
496 + marlin_s = marlin_permute_scales(scale.transpose(1, 0), size_k, size_n,
497 + group_size).to(dev)
498 + marlin_zp = marlin_permute_scales(zero.transpose(1, 0), size_k, size_n,
499 + group_size).to(dev)
500 +
501 + g_idx = marlin_make_empty_g_idx(dev)
502 + g_idx_sort_indices = marlin_make_empty_g_idx(dev)
503 +
504 + workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
505 + GPTQ_MARLIN_MAX_PARALLEL)
506 +
507 + output = ops.gptq_marlin_gemm(
508 + a_input,
509 + marlin_w_q,
510 + marlin_s,
511 + marlin_zp,
512 + g_idx,
513 + g_idx_sort_indices,
514 + workspace.scratch,
515 + quant_type,
516 + a_input.shape[0],
517 + b_weight.shape[0],
518 + a_input.shape[1],
519 + is_k_full=True,
520 + has_zp=True,
521 + use_fp32_reduce=use_fp32_reduce,
522 + is_zp_float=True,
523 + )
524 +
525 + b_flat = b_weight.reshape(-1, group_size)
526 + zp_flat = zero.reshape(-1, 1)
527 + s_flat = scale.reshape(-1, 1)
528 + dequant = (b_flat - zp_flat) * s_flat
529 +
530 + output_ref = torch.matmul(a_input,
531 + dequant.reshape(b_weight.shape).transpose(1, 0))
532 +
533 + torch.cuda.synchronize()
534 +
535 + max_diff = compute_max_diff(output, output_ref)
536 +
537 + assert max_diff < 0.04
538 +
539 +
540 @pytest.mark.skipif(not is_quant_method_supported("qqq"),
541 reason="Marlin is not supported on this GPU type.")
542 @pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
tests/weight_loading/models.txt
+2 -1
@@ -27,4 +27,5 @@ fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
27 marlin, nm-testing/zephyr-beta-7b-marlin-g128, main
28 marlin, robertgshaw2/zephyr-7b-beta-channelwise-marlin, main
29 qqq, HandH1998/QQQ-Llama-3-8b-g128, main
30 -qqq, HandH1998/QQQ-Llama-3-8b, main
\ No newline at end of file
30 +qqq, HandH1998/QQQ-Llama-3-8b, main
31 +hqq, nm-testing/Llama-3.2-1B-Instruct-HQQ, main
\ No newline at end of file
vllm/_custom_ops.py
+5 -3
@@ -343,7 +343,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
343 size_k: torch.SymInt,
344 is_k_full: bool,
345 has_zp: bool = False,
346 - use_fp32_reduce: bool = False) -> torch.Tensor:
346 + use_fp32_reduce: bool = False,
347 + is_zp_float: bool = False) -> torch.Tensor:
348 return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
349
350 @register_fake("_C::ggml_dequantize")
@@ -601,11 +602,12 @@ def gptq_marlin_gemm(a: torch.Tensor,
602 size_k: int,
603 is_k_full: bool,
604 has_zp: bool = False,
604 - use_fp32_reduce: bool = False) -> torch.Tensor:
605 + use_fp32_reduce: bool = False,
606 + is_zp_float: bool = False) -> torch.Tensor:
607 return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
608 g_idx, perm, workspace, b_q_type.id,
609 size_m, size_n, size_k, is_k_full,
608 - has_zp, use_fp32_reduce)
610 + has_zp, use_fp32_reduce, is_zp_float)
611
612
613 # fp8 marlin
vllm/model_executor/layers/linear.py
+2 -1
@@ -27,7 +27,8 @@ WEIGHT_LOADER_V2_SUPPORTED = [
27 "AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28 "MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
29 "TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod",
30 - "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod"
30 + "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod",
31 + "HQQMarlinMethod"
32 ]
33
34
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
21 GPTQMarlinConfig)
22 from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
23 GPTQMarlin24Config)
24 +from vllm.model_executor.layers.quantization.hqq_marlin import HQQMarlinConfig
25 from vllm.model_executor.layers.quantization.ipex_quant import IPEXConfig
26 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
27 from vllm.model_executor.layers.quantization.modelopt import ModelOptFp8Config
@@ -48,6 +49,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
49 "compressed-tensors": CompressedTensorsConfig,
50 "bitsandbytes": BitsAndBytesConfig,
51 "qqq": QQQConfig,
52 + "hqq": HQQMarlinConfig,
53 "experts_int8": ExpertsInt8Config,
54 "neuron_quant": NeuronQuantConfig,
55 "ipex": IPEXConfig,
vllm/model_executor/layers/quantization/hqq_marlin.py new
+325
@@ -0,0 +1,325 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +
5 +from vllm import _custom_ops as ops
6 +from vllm.logger import init_logger
7 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
8 + UnquantizedLinearMethod)
9 +from vllm.model_executor.layers.quantization.base_config import (
10 + QuantizationConfig, QuantizeMethodBase)
11 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13 + marlin_make_empty_g_idx, marlin_permute_scales)
14 +from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15 + MarlinWorkspace)
16 +from vllm.model_executor.layers.quantization.utils.quant_utils import gptq_pack
17 +from vllm.model_executor.parameter import (BasevLLMParameter,
18 + GroupQuantScaleParameter,
19 + PackedvLLMParameter)
20 +from vllm.scalar_type import scalar_types
21 +
22 +logger = init_logger(__name__)
23 +
24 +
25 +class HQQMarlinConfig(QuantizationConfig):
26 + """Config class for HQQ Marlin"""
27 +
28 + def __init__(
29 + self,
30 + weight_bits: int,
31 + group_size: int,
32 + skip_modules: Optional[List[str]] = None,
33 + ) -> None:
34 + assert group_size == 64, ("The only supported HQQ group size is "
35 + "currently 64.")
36 + assert weight_bits == 4, ("The only supported HQQ quantization "
37 + "bitsize is currently 4.")
38 +
39 + self.weight_bits = weight_bits
40 + self.group_size = group_size
41 + self.pack_factor = 32 // weight_bits # packed into int32 in GPTQ format
42 + self.quant_type = scalar_types.uint4
43 + self.skip_modules = skip_modules
44 +
45 + def __repr__(self) -> str:
46 + return (f"HQQMarlinConfig(quant_type={self.quant_type}, "
47 + f"group_size={self.group_size})")
48 +
49 + @classmethod
50 + def get_name(cls) -> str:
51 + return "hqq"
52 +
53 + @classmethod
54 + def get_supported_act_dtypes(cls) -> List[torch.dtype]:
55 + return [torch.half, torch.bfloat16]
56 +
57 + @classmethod
58 + def get_min_capability(cls) -> int:
59 + return 80
60 +
61 + @classmethod
62 + def get_config_filenames(cls) -> List[str]:
63 + return ["quantize_config.json"]
64 +
65 + @classmethod
66 + def from_config(cls, config: Dict[str, Any]) -> "HQQMarlinConfig":
67 + wq_params = (config["quant_config"]["weight_quant_params"])
68 + weight_bits = cls.get_from_keys(wq_params, ["nbits"])
69 + group_size = cls.get_from_keys(wq_params, ["group_size"])
70 + skip_modules = config["skip_modules"]
71 + return cls(weight_bits, group_size, skip_modules)
72 +
73 + def is_layer_skipped(self, prefix: str) -> bool:
74 + # Split the prefix into its dot-separated components
75 + components = prefix.split('.')
76 +
77 + # Check if any of the skip modules exactly matches any component
78 + return self.skip_modules is not None and any(
79 + module_name in components for module_name in self.skip_modules)
80 +
81 + def get_quant_method(self, layer: torch.nn.Module,
82 + prefix: str) -> Optional["QuantizeMethodBase"]:
83 + if isinstance(layer, LinearBase):
84 + if self.is_layer_skipped(prefix):
85 + return UnquantizedLinearMethod()
86 + return HQQMarlinMethod(self)
87 + return None
88 +
89 +
90 +# Empty HQQ parameter, will be ignored during loading
91 +class HQQEmptyParameter(BasevLLMParameter):
92 +
93 + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
94 + pass
95 +
96 + def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
97 + pass
98 +
99 + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
100 + pass
101 +
102 +
103 +def error_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
104 + raise ValueError("No loader provided for HQQ parameter!")
105 +
106 +
107 +# HQQ packing creates issues with sharding - therefore, prior to loading, we
108 +# repack to GPTQ. We also reshape the weights to their proper GPTQ shape.
109 +class HQQweightParameter(PackedvLLMParameter):
110 +
111 + # unpack function from https://github.com/mobiusml/hqq
112 + def unpack_4bit_u8(self,
113 + W_q: torch.Tensor) -> torch.Tensor: # uint8/2 > uint8
114 + assert self.weight_bits == 4, "Unsupported quant bitsize (must be 4)"
115 +
116 + dtype = torch.uint8
117 + step = W_q.shape[0]
118 + tmp = torch.empty([2 * step, W_q.shape[1]],
119 + dtype=dtype,
120 + device=W_q.device)
121 + tmp[:step] = (W_q & 0b11110000) >> 4
122 + tmp[step:] = W_q & 0b00001111
123 + return tmp
124 +
125 + def __init__(self, packed_factor: int, packed_dim: int, weight_bits: int,
126 + **kwargs):
127 + super().__init__(packed_factor, packed_dim, None, **kwargs)
128 + self.weight_bits = weight_bits
129 + self.input_shape = self.shape[self.input_dim] * self.packed_factor
130 + self.output_shape = self.shape[self.output_dim]
131 +
132 + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
133 + loaded_weight = self.unpack_4bit_u8(loaded_weight)
134 + loaded_weight = loaded_weight.reshape(-1, self.input_shape).transpose(
135 + 1, 0)
136 + loaded_weight = gptq_pack(loaded_weight, self.weight_bits,
137 + loaded_weight.shape[0],
138 + loaded_weight.shape[1])
139 + super().load_merged_column_weight(loaded_weight, **kwargs)
140 +
141 + def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
142 + loaded_weight = self.unpack_4bit_u8(loaded_weight)
143 + loaded_weight = loaded_weight.reshape(self.output_shape,
144 + -1).transpose(1, 0)
145 + loaded_weight = gptq_pack(loaded_weight, self.weight_bits,
146 + loaded_weight.shape[0],
147 + loaded_weight.shape[1])
148 + super().load_row_parallel_weight(loaded_weight)
149 +
150 + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
151 + loaded_weight = self.unpack_4bit_u8(loaded_weight)
152 + loaded_weight = loaded_weight.reshape(-1, self.input_shape).transpose(
153 + 1, 0)
154 + loaded_weight = gptq_pack(loaded_weight, self.weight_bits,
155 + loaded_weight.shape[0],
156 + loaded_weight.shape[1])
157 + super().load_qkv_weight(loaded_weight, **kwargs)
158 +
159 +
160 +# Zero points and scales in HQQ must also be reshaped to correspond to W_q's
161 +# GPTQ shape (transposed - we transpose them too when processing weights).
162 +class HQQZeroScaleParameter(GroupQuantScaleParameter):
163 +
164 + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
165 + loaded_weight = loaded_weight.reshape(-1, self.shape[1])
166 + super().load_merged_column_weight(loaded_weight, **kwargs)
167 +
168 + def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
169 + loaded_weight = loaded_weight.reshape(self.shape[0], -1)
170 + super().load_row_parallel_weight(loaded_weight)
171 +
172 + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
173 + loaded_weight = loaded_weight.reshape(-1, self.shape[1])
174 + super().load_qkv_weight(loaded_weight, **kwargs)
175 +
176 +
177 +class HQQMarlinMethod(LinearMethodBase):
178 + """Linear method for HQQ Marlin.
179 + """
180 +
181 + def __init__(
182 + self,
183 + quant_config: HQQMarlinConfig,
184 + ):
185 + self.quant_config = quant_config
186 +
187 + def create_weights(
188 + self,
189 + layer: torch.nn.Module,
190 + input_size_per_partition: int,
191 + output_partition_sizes: List[int],
192 + input_size: int,
193 + output_size: int,
194 + params_dtype: torch.dtype,
195 + **extra_weight_attrs,
196 + ) -> None:
197 + self.output_size_per_partition = sum(output_partition_sizes)
198 + self.input_size_per_partition = input_size_per_partition
199 +
200 + weight_loader = extra_weight_attrs.get("weight_loader", error_loader)
201 +
202 + self.scales_and_zp_size = (input_size_per_partition //
203 + self.quant_config.group_size)
204 +
205 + qweight = HQQweightParameter(
206 + data=torch.empty(
207 + self.input_size_per_partition // self.quant_config.pack_factor,
208 + self.output_size_per_partition,
209 + dtype=torch.int32,
210 + ),
211 + input_dim=0,
212 + output_dim=1,
213 + packed_dim=0,
214 + packed_factor=self.quant_config.pack_factor,
215 + weight_bits=self.quant_config.weight_bits,
216 + weight_loader=weight_loader)
217 +
218 + zeros = HQQZeroScaleParameter(data=torch.empty(
219 + self.output_size_per_partition,
220 + self.scales_and_zp_size,
221 + dtype=params_dtype,
222 + ),
223 + input_dim=1,
224 + output_dim=0,
225 + weight_loader=weight_loader)
226 +
227 + scales = HQQZeroScaleParameter(data=torch.empty(
228 + self.output_size_per_partition,
229 + self.scales_and_zp_size,
230 + dtype=params_dtype,
231 + ),
232 + input_dim=1,
233 + output_dim=0,
234 + weight_loader=weight_loader)
235 +
236 + layer.register_parameter("W_q", qweight)
237 + layer.register_parameter("zero", zeros)
238 + layer.register_parameter("scale", scales)
239 +
240 + # Ignore extra parameters in the HQQ model.
241 + # To be added as needed.
242 + ignore_parameters = ("axis", "channel_wise", "compute_dtype",
243 + "encoded_state_dict", "group_size", "nbits",
244 + "offload_meta", "optimize", "packing",
245 + "quant_scale", "quant_zero", "round_zero",
246 + "shape", "stores_quant_config",
247 + "unpack_view_dtype", "view_as_float")
248 + for name in ignore_parameters:
249 + layer.register_parameter(
250 + name,
251 + HQQEmptyParameter(data=torch.empty(0),
252 + weight_loader=weight_loader))
253 +
254 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
255 + dev = layer.W_q.device
256 +
257 + # Repack to Marlin
258 + sort_indices = torch.empty(0, dtype=torch.int, device=dev)
259 + marlin_w_q = ops.gptq_marlin_repack(
260 + layer.W_q,
261 + sort_indices,
262 + self.input_size_per_partition,
263 + self.output_size_per_partition,
264 + self.quant_config.weight_bits,
265 + ).to(dev)
266 + marlin_s = marlin_permute_scales(layer.scale.transpose(1, 0),
267 + self.input_size_per_partition,
268 + self.output_size_per_partition,
269 + self.quant_config.group_size).to(dev)
270 + marlin_zp = marlin_permute_scales(layer.zero.transpose(1, 0),
271 + self.input_size_per_partition,
272 + self.output_size_per_partition,
273 + self.quant_config.group_size).to(dev)
274 +
275 + layer.g_idx = marlin_make_empty_g_idx(dev)
276 + layer.g_idx_sort_indices = marlin_make_empty_g_idx(dev)
277 +
278 + layer.marlin_qweight = marlin_w_q
279 + layer.marlin_zeros = marlin_zp
280 + layer.marlin_scales = marlin_s
281 +
282 + def apply(
283 + self,
284 + layer: torch.nn.Module,
285 + x: torch.Tensor,
286 + bias: Optional[torch.Tensor] = None,
287 + ) -> torch.Tensor:
288 + workspace = MarlinWorkspace(self.output_size_per_partition,
289 + GPTQ_MARLIN_MIN_THREAD_N,
290 + GPTQ_MARLIN_MAX_PARALLEL)
291 +
292 + scales = layer.marlin_scales
293 + zeros = layer.marlin_zeros
294 + orig_type = x.dtype
295 +
296 + if orig_type != torch.float16:
297 + x = x.to(torch.float16)
298 + scales = scales.to(torch.float16)
299 + zeros = zeros.to(torch.float16)
300 +
301 + marlin_out = ops.gptq_marlin_gemm(
302 + x,
303 + layer.marlin_qweight,
304 + scales,
305 + zeros,
306 + layer.g_idx,
307 + layer.g_idx_sort_indices,
308 + workspace.scratch,
309 + scalar_types.uint4,
310 + x.shape[0],
311 + self.output_size_per_partition,
312 + self.input_size_per_partition,
313 + True, # is_k_full
314 + True, # has_zp
315 + True, # use 32-bit reduce
316 + True, # use float zp
317 + )
318 +
319 + if orig_type != torch.float16:
320 + marlin_out = marlin_out.to(orig_type)
321 +
322 + if bias is not None:
323 + marlin_out.add_(bias)
324 +
325 + return marlin_out
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+4 -2
@@ -303,7 +303,8 @@ def apply_gptq_marlin_linear(
303 size_k=input_size_per_partition,
304 is_k_full=is_k_full,
305 has_zp=False,
306 - use_fp32_reduce=use_fp32_reduce)
306 + use_fp32_reduce=use_fp32_reduce,
307 + is_zp_float=False)
308
309 if bias is not None:
310 output.add_(bias) # In-place add
@@ -340,7 +341,8 @@ def apply_awq_marlin_linear(
341 size_k=input_size_per_partition,
342 is_k_full=True,
343 has_zp=True,
343 - use_fp32_reduce=use_fp32_reduce)
344 + use_fp32_reduce=use_fp32_reduce,
345 + is_zp_float=False)
346
347 if bias is not None:
348 output.add_(bias) # In-place add