A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model][Quantization] HQQ support through Marlin kernel expansion (#9766)
Signed-off-by: ElizaWszola <eliza@neuralmagic.com>
ElizaWszola committed
Nov 19, 2024 at 22:31 UTC
b00b33d77e33c5516e73de663539dff96e8b61a4
11 files changed
+632
-89
benchmarks/kernels/benchmark_machete.py
+2
-1
@@ -210,7 +210,8 @@ def marlin_create_bench_fn(bt: BenchmarkTensors) -> Callable:
210
size_m=bt.a.shape[0],
211
size_n=bt.w_ref.shape[1],
212
size_k=bt.w_ref.shape[0],
213
- is_k_full=True)
213
+ is_k_full=True,
214
+ is_zp_float=False)
215
else:
216
assert bt.a.dtype == torch.int8
217
assert bt.wtype == scalar_types.uint4b8
benchmarks/kernels/benchmark_marlin.py
+2
-2
@@ -131,7 +131,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
131
results.append(
132
benchmark.Timer(
133
stmt=
134
- "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
134
+ "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, False, False)", # noqa: E501
135
globals=globals,
136
label=label,
137
sub_label=sub_label,
@@ -141,7 +141,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
141
results.append(
142
benchmark.Timer(
143
stmt=
144
- "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
144
+ "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, quant_type, size_m, size_n, size_k, is_k_full, False, True, False)", # noqa: E501
145
globals=globals,
146
label=label,
147
sub_label=sub_label,
csrc/quantization/gptq_marlin/gptq_marlin.cu
+200
-77
@@ -54,9 +54,10 @@ template <typename scalar_t, // compute dtype, half or nv_float16
54
const int thread_k_blocks, // same for k dimension (reduction)
55
const int stages, // number of stages for the async global->shared
56
// fetch pipeline
57
- const bool has_act_order, // whether act_order is enabled
58
- const int group_blocks = -1 // number of consecutive 16x16 blocks
59
- // with a separate quantization scale
57
+ const bool has_act_order, // whether act_order is enabled
58
+ const int group_blocks = -1, // number of consecutive 16x16 blocks
59
+ // with a separate quantization scale
60
+ const bool is_zp_float // is zero point of float16 type?
61
>
62
__global__ void Marlin(
63
const int4* __restrict__ A, // fp16 input matrix of shape mxk
@@ -82,7 +83,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
83
torch::Tensor& workspace,
84
vllm::ScalarTypeId const b_q_type_id,
85
int64_t size_m, int64_t size_n, int64_t size_k,
85
- bool is_k_full, bool has_zp) {
86
+ bool is_k_full, bool has_zp, bool is_zp_float) {
87
TORCH_CHECK_NOT_IMPLEMENTED(false,
88
"marlin_gemm(..) requires CUDA_ARCH >= 8.0");
89
return torch::empty({1, 1});
@@ -516,10 +517,11 @@ template <typename scalar_t, // compute dtype, half or nv_float16
517
const int thread_k_blocks, // same for k dimension (reduction)
518
const int stages, // number of stages for the async global->shared
519
// fetch pipeline
519
- const bool has_act_order, // whether act_order is enabled
520
- const bool has_zp, // whether zero-points are enabled
521
- const int group_blocks = -1 // number of consecutive 16x16 blocks
522
- // with a separate quantization scale
520
+ const bool has_act_order, // whether act_order is enabled
521
+ const bool has_zp, // whether zero-points are enabled
522
+ const int group_blocks = -1, // number of consecutive 16x16 blocks
523
+ // with a separate quantization scale
524
+ const bool is_zp_float // is zero point of float16 type?
525
>
526
__global__ void Marlin(
527
const int4* __restrict__ A, // fp16 input matrix of shape mxk
@@ -692,8 +694,10 @@ __global__ void Marlin(
694
int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
695
696
// Zero-points sizes/strides
695
- int zp_gl_stride = (prob_n / pack_factor) / 4;
696
- constexpr int zp_sh_stride = ((16 * thread_n_blocks) / pack_factor) / 4;
697
+ int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4;
698
+ constexpr int zp_sh_stride = is_zp_float
699
+ ? 16 * thread_n_blocks / 8
700
+ : ((16 * thread_n_blocks) / pack_factor) / 4;
701
constexpr int zp_tb_groups = s_tb_groups;
702
constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0;
703
int zp_gl_rd_delta = zp_gl_stride;
@@ -768,9 +772,16 @@ __global__ void Marlin(
772
constexpr int num_ints_per_thread = 8 / pack_factor;
773
int zp_sh_rd;
774
if constexpr (has_zp) {
771
- zp_sh_rd = num_ints_per_thread * num_col_threads *
772
- ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
773
- num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
775
+ if constexpr (is_zp_float) {
776
+ if constexpr (group_blocks != -1) {
777
+ zp_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
778
+ (threadIdx.x % 32) / 4;
779
+ }
780
+ } else {
781
+ zp_sh_rd = num_ints_per_thread * num_col_threads *
782
+ ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
783
+ num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads);
784
+ }
785
}
786
787
// Precompute which thread should not read memory in which iterations; this is
@@ -832,6 +843,7 @@ __global__ void Marlin(
843
FragS act_frag_s[2][4][4]; // For act-order
844
int frag_qzp[2][num_ints_per_thread]; // Zero-points
845
FragZP frag_zp; // Zero-points in fp16
846
+ FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ
847
848
// Zero accumulators.
849
auto zero_accums = [&]() {
@@ -1126,7 +1138,7 @@ __global__ void Marlin(
1138
// has_zp implies AWQ, which doesn't have act_order,
1139
static_assert(!has_zp || group_blocks != 0);
1140
1129
- if constexpr (has_zp) {
1141
+ if constexpr (has_zp && !is_zp_float) {
1142
int pipe = full_pipe % stages;
1143
1144
if constexpr (group_blocks == -1) {
@@ -1170,11 +1182,44 @@ __global__ void Marlin(
1182
}
1183
}
1184
}
1185
+
1186
+ else if constexpr (has_zp && is_zp_float) {
1187
+ int pipe = full_pipe % stages;
1188
+
1189
+ if constexpr (group_blocks != -1) {
1190
+ if constexpr (group_blocks >= thread_k_blocks) {
1191
+ int4* sh_zp_stage =
1192
+ sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1193
+ (pipe / (group_blocks / thread_k_blocks)));
1194
+ reinterpret_cast<int4*>(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd];
1195
+ } else {
1196
+ int warp_id = threadIdx.x / 32;
1197
+ int n_warps = thread_n_blocks / 4;
1198
+
1199
+ int warp_row = warp_id / n_warps;
1200
+
1201
+ int cur_k = warp_row * 16;
1202
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
1203
+
1204
+ int k_blocks = cur_k / 16;
1205
+ // Suppress bogus and persistent divide-by-zero warning
1206
+ #pragma nv_diagnostic push
1207
+ #pragma nv_diag_suppress divide_by_zero
1208
+ int cur_group_id = k_blocks / group_blocks;
1209
+ #pragma nv_diagnostic pop
1210
+
1211
+ int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1212
+
1213
+ reinterpret_cast<int4*>(&frag_zpf[k % 2])[0] =
1214
+ sh_zp_stage[zp_sh_rd + cur_group_id * zp_sh_stride];
1215
+ }
1216
+ }
1217
+ }
1218
};
1219
1220
// Execute the actual tensor core matmul of a sub-tile.
1221
auto matmul = [&](int k) {
1177
- if constexpr (has_zp) {
1222
+ if constexpr (has_zp && !is_zp_float) {
1223
FragB frag_zp_0;
1224
FragB frag_zp_1;
1225
int zp_quant_0, zp_quant_1;
@@ -1219,10 +1264,14 @@ __global__ void Marlin(
1264
frag_b1 = dequant<scalar_t, w_type_id>(b_quant_1);
1265
1266
// Apply zero-point to frag_b0
1222
- if constexpr (has_zp) {
1267
+ if constexpr (has_zp && !is_zp_float) {
1268
sub_zp<scalar_t>(frag_b0, frag_zp[j], 0);
1269
}
1270
1271
+ else if constexpr (has_zp && is_zp_float && group_blocks != -1) {
1272
+ sub_zp<scalar_t>(frag_b0, frag_zpf[k % 2][j], 0);
1273
+ }
1274
+
1275
// Apply scale to frag_b0
1276
if constexpr (has_act_order) {
1277
scale4<scalar_t>(frag_b0, act_frag_s[k % 2][0][j],
@@ -1235,10 +1284,14 @@ __global__ void Marlin(
1284
}
1285
1286
// Apply zero-point to frag_b1
1238
- if constexpr (has_zp) {
1287
+ if constexpr (has_zp && !is_zp_float) {
1288
sub_zp<scalar_t>(frag_b1, frag_zp[j], 1);
1289
}
1290
1291
+ else if constexpr (has_zp && is_zp_float && group_blocks != -1) {
1292
+ sub_zp<scalar_t>(frag_b1, frag_zpf[k % 2][j], 1);
1293
+ }
1294
+
1295
// Apply scale to frag_b1
1296
if constexpr (has_act_order) {
1297
scale4<scalar_t>(frag_b1, act_frag_s[k % 2][0][j],
@@ -1510,7 +1563,7 @@ __global__ void Marlin(
1563
fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1564
}
1565
1513
- if constexpr (has_zp && group_blocks == -1) {
1566
+ if constexpr (has_zp && !is_zp_float && group_blocks == -1) {
1567
if (i == 0) {
1568
fetch_zp_to_shared();
1569
}
@@ -1697,23 +1750,27 @@ __global__ void Marlin(
1750
}
1751
1752
#define __CALL_IF(W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1700
- HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, NUM_THREADS) \
1753
+ HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, NUM_THREADS, \
1754
+ IS_ZP_FLOAT) \
1755
else if (q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && \
1756
thread_n_blocks == THREAD_N_BLOCKS && \
1757
thread_k_blocks == THREAD_K_BLOCKS && \
1758
has_act_order == HAS_ACT_ORDER && has_zp == HAS_ZP && \
1705
- group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
1706
- cudaFuncSetAttribute( \
1707
- Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1708
- THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1709
- HAS_ZP, GROUP_BLOCKS>, \
1710
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1711
- Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1712
- THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1713
- HAS_ZP, GROUP_BLOCKS> \
1714
- <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1715
- A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, zp_ptr, g_idx_ptr, \
1716
- num_groups, prob_m, prob_n, prob_k, locks, use_fp32_reduce); \
1759
+ group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS && \
1760
+ is_zp_float == IS_ZP_FLOAT) { \
1761
+ if constexpr (!IS_ZP_FLOAT || std::is_same<scalar_t, half>::value) { \
1762
+ cudaFuncSetAttribute( \
1763
+ Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1764
+ THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, \
1765
+ HAS_ACT_ORDER, HAS_ZP, GROUP_BLOCKS, IS_ZP_FLOAT>, \
1766
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1767
+ Marlin<scalar_t, W_TYPE.id(), NUM_THREADS, THREAD_M_BLOCKS, \
1768
+ THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1769
+ HAS_ZP, GROUP_BLOCKS, IS_ZP_FLOAT> \
1770
+ <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1771
+ A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, zp_ptr, g_idx_ptr, \
1772
+ num_groups, prob_m, prob_n, prob_k, locks, use_fp32_reduce); \
1773
+ } \
1774
}
1775
1776
typedef struct {
@@ -1905,51 +1962,96 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1962
}
1963
1964
#define GPTQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1908
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1909
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1910
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1911
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS) \
1965
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1966
+ false) \
1967
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1968
+ false) \
1969
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1970
+ false) \
1971
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, true, false, 0, NUM_THREADS, \
1972
+ false) \
1973
\
1913
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1914
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1915
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1916
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1974
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
1975
+ false) \
1976
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
1977
+ false) \
1978
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
1979
+ false) \
1980
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
1981
+ false) \
1982
\
1918
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1919
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1920
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1921
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1983
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
1984
+ false) \
1985
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
1986
+ false) \
1987
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
1988
+ false) \
1989
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
1990
+ false) \
1991
\
1923
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1924
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1925
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1926
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS) \
1992
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
1993
+ false) \
1994
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
1995
+ false) \
1996
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
1997
+ false) \
1998
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
1999
+ false) \
2000
\
1928
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS) \
1929
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS) \
1930
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS) \
1931
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS)
2001
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, -1, NUM_THREADS, \
2002
+ false) \
2003
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 2, NUM_THREADS, \
2004
+ false) \
2005
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 4, NUM_THREADS, \
2006
+ false) \
2007
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, 8, NUM_THREADS, \
2008
+ false)
2009
2010
#define AWQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1934
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1935
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1936
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1937
- __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
2011
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2012
+ false) \
2013
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2014
+ false) \
2015
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2016
+ false) \
2017
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, \
2018
+ false) \
2019
\
1939
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1940
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1941
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1942
- __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
2020
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2021
+ false) \
2022
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2023
+ false) \
2024
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2025
+ false) \
2026
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, \
2027
+ false) \
2028
\
1944
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1945
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1946
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1947
- __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS) \
2029
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2030
+ false) \
2031
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2032
+ false) \
2033
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2034
+ false) \
2035
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, \
2036
+ false) \
2037
\
1949
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS) \
1950
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS) \
1951
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS) \
1952
- __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
2038
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, -1, NUM_THREADS, \
2039
+ false) \
2040
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 2, NUM_THREADS, \
2041
+ false) \
2042
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2043
+ false) \
2044
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS, false)
2045
+
2046
+ // We currently have 4-bit models only with group_blocks == 4
2047
+ #define HQQ_CALL_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
2048
+ __CALL_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2049
+ true) \
2050
+ __CALL_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2051
+ true) \
2052
+ __CALL_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, \
2053
+ true) \
2054
+ __CALL_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, 4, NUM_THREADS, true)
2055
2056
template <typename scalar_t>
2057
void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
@@ -1958,7 +2060,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
2060
vllm::ScalarType const& q_type, bool has_act_order,
2061
bool is_k_full, bool has_zp, int num_groups, int group_size,
2062
int dev, cudaStream_t stream, int thread_k, int thread_n,
1961
- int sms, int max_par, bool use_fp32_reduce) {
2063
+ int sms, int max_par, bool use_fp32_reduce, bool is_zp_float) {
2064
if (has_zp) {
2065
TORCH_CHECK(
2066
q_type == vllm::kU4 || q_type == vllm::kU8,
@@ -2111,6 +2213,11 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* s,
2213
AWQ_CALL_IF(vllm::kU8, 8, 8, 256)
2214
AWQ_CALL_IF(vllm::kU8, 8, 4, 128)
2215
AWQ_CALL_IF(vllm::kU8, 4, 8, 128)
2216
+
2217
+ HQQ_CALL_IF(vllm::kU4, 16, 4, 256)
2218
+ HQQ_CALL_IF(vllm::kU4, 8, 8, 256)
2219
+ HQQ_CALL_IF(vllm::kU4, 8, 4, 128)
2220
+ HQQ_CALL_IF(vllm::kU4, 4, 8, 128)
2221
else {
2222
TORCH_CHECK(false, "Unsupported shapes: MNK = [", prob_m, ", ", prob_n,
2223
", ", prob_k, "]", ", has_act_order = ", has_act_order,
@@ -2135,7 +2242,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2242
vllm::ScalarTypeId const& b_q_type_id,
2243
int64_t size_m, int64_t size_n, int64_t size_k,
2244
bool is_k_full, bool has_zp,
2138
- bool use_fp32_reduce) {
2245
+ bool use_fp32_reduce, bool is_zp_float) {
2246
vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
2247
if (has_zp) {
2248
TORCH_CHECK(
@@ -2148,6 +2255,12 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2255
b_q_type.str());
2256
}
2257
2258
+ if (has_zp && is_zp_float) {
2259
+ TORCH_CHECK(a.scalar_type() == at::ScalarType::Half,
2260
+ "Computation type must be float16 (half) when using float zero "
2261
+ "points.");
2262
+ }
2263
+
2264
int pack_factor = 32 / b_q_type.size_bits();
2265
2266
// Verify A
@@ -2257,12 +2370,22 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2370
if (has_zp) {
2371
int rank = b_zeros.sizes().size();
2372
TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2");
2260
- TORCH_CHECK(b_zeros.size(0) == num_groups,
2261
- "b_zeros dim 0 = ", b_zeros.size(0),
2262
- " is not num_groups = ", num_groups);
2263
- TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2264
- "b_zeros dim 1 = ", b_zeros.size(1),
2265
- " is not size_n / pack_factor = ", size_n / pack_factor);
2373
+ if (is_zp_float) {
2374
+ TORCH_CHECK(b_zeros.size(1) == size_n,
2375
+ "b_zeros dim 1 = ", b_zeros.size(1),
2376
+ " is not size_n = ", size_n);
2377
+ TORCH_CHECK(num_groups == b_zeros.size(0),
2378
+ "b_zeros dim 0 = ", b_zeros.size(0),
2379
+ " is not num_groups = ", num_groups);
2380
+ TORCH_CHECK(num_groups != -1, "num_groups must be != -1");
2381
+ } else {
2382
+ TORCH_CHECK(b_zeros.size(0) == num_groups,
2383
+ "b_zeros dim 0 = ", b_zeros.size(0),
2384
+ " is not num_groups = ", num_groups);
2385
+ TORCH_CHECK(b_zeros.size(1) == size_n / pack_factor,
2386
+ "b_zeros dim 1 = ", b_zeros.size(1),
2387
+ " is not size_n / pack_factor = ", size_n / pack_factor);
2388
+ }
2389
}
2390
2391
// Verify workspace size
@@ -2282,7 +2405,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2405
a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2406
workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2407
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2285
- thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2408
+ thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce, is_zp_float);
2409
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
2410
marlin::marlin_mm<nv_bfloat16>(
2411
a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
@@ -2291,7 +2414,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2414
perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2415
workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2416
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2294
- thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2417
+ thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce, is_zp_float);
2418
} else {
2419
TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
2420
}
csrc/torch_bindings.cpp
+1
-1
@@ -244,7 +244,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
244
"Tensor b_zeros, Tensor g_idx, Tensor perm, Tensor workspace, "
245
"int b_q_type, "
246
"SymInt size_m, SymInt size_n, SymInt size_k, bool is_k_full, "
247
- "bool has_zp, bool use_fp32_reduce) -> Tensor");
247
+ "bool has_zp, bool use_fp32_reduce, bool is_zp_float) -> Tensor");
248
// conditionally compiled so impl registration is in source file
249
250
// gptq_marlin repack from GPTQ.
tests/kernels/test_marlin_gemm.py
+87
-1
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_test_qqq import
29
marlin_qqq_quantize)
30
from vllm.model_executor.layers.quantization.utils.quant_utils import (
31
awq_pack, gptq_pack, gptq_quantize_weights, quantize_weights, sort_weights)
32
+from vllm.scalar_type import scalar_types
33
34
ACT_ORDER_OPTS = [False, True]
35
K_FULL_OPTS = [False, True]
@@ -40,6 +41,8 @@ MARLIN_N_CHUNKS = [64, 256]
41
MARLIN_24_K_CHUNKS = [128]
42
MARLIN_24_N_CHUNKS = [512]
43
44
+HQQ_SUPPORTED_GROUP_SIZES = [64]
45
+
46
MNK_FACTORS = [
47
(1, 1, 1),
48
(1, 4, 8),
@@ -226,7 +229,7 @@ def test_gptq_marlin_gemm(
229
torch.ops._C.gptq_marlin_gemm,
230
(a_input, marlin_q_w, marlin_s, marlin_zp, g_idx, sort_indices,
231
workspace.scratch, quant_type.id, a_input.shape[0], b_weight.shape[1],
229
- a_input.shape[1], is_k_full, False, use_fp32_reduce),
232
+ a_input.shape[1], is_k_full, False, use_fp32_reduce, False),
233
test_utils=DEFAULT_OPCHECK_TEST_UTILS)
234
235
output = ops.gptq_marlin_gemm(
@@ -244,6 +247,7 @@ def test_gptq_marlin_gemm(
247
is_k_full=is_k_full,
248
has_zp=False,
249
use_fp32_reduce=use_fp32_reduce,
250
+ is_zp_float=False,
251
)
252
output_ref = torch.matmul(a_input, w_ref)
253
@@ -441,6 +445,7 @@ def test_awq_marlin_gemm(
445
is_k_full=is_k_full,
446
has_zp=has_zp,
447
use_fp32_reduce=use_fp32_reduce,
448
+ is_zp_float=False,
449
)
450
output_ref = torch.matmul(a_input, w_ref)
451
@@ -451,6 +456,87 @@ def test_awq_marlin_gemm(
456
assert max_diff < 0.04
457
458
459
+@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
460
+ reason="Marlin is not supported on this GPU type.")
461
+@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
462
+@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
463
+@pytest.mark.parametrize("group_size", HQQ_SUPPORTED_GROUP_SIZES)
464
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
465
+@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
466
+def test_hqq_marlin_gemm(
467
+ k_chunk,
468
+ n_chunk,
469
+ group_size,
470
+ mnk_factors,
471
+ use_fp32_reduce,
472
+):
473
+ m_factor, n_factor, k_factor = mnk_factors
474
+
475
+ size_m = m_factor
476
+ size_k = k_chunk * k_factor
477
+ size_n = n_chunk * n_factor
478
+
479
+ quant_type = scalar_types.uint4
480
+
481
+ a_input = rand_data((size_m, size_k))
482
+ dev = a_input.device
483
+
484
+ b_weight = torch.randint(0,
485
+ 10, (size_n, size_k),
486
+ dtype=torch.uint8,
487
+ device=dev)
488
+ scale = rand_data((size_n, size_k // group_size))
489
+ zero = rand_data((size_n, size_k // group_size))
490
+
491
+ gptq_w_q = gptq_pack(b_weight.transpose(1, 0), 4, size_k, size_n)
492
+
493
+ sort_indices = torch.empty(0, dtype=torch.int, device=dev)
494
+ marlin_w_q = ops.gptq_marlin_repack(gptq_w_q, sort_indices, size_k, size_n,
495
+ 4).to(dev)
496
+ marlin_s = marlin_permute_scales(scale.transpose(1, 0), size_k, size_n,
497
+ group_size).to(dev)
498
+ marlin_zp = marlin_permute_scales(zero.transpose(1, 0), size_k, size_n,
499
+ group_size).to(dev)
500
+
501
+ g_idx = marlin_make_empty_g_idx(dev)
502
+ g_idx_sort_indices = marlin_make_empty_g_idx(dev)
503
+
504
+ workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
505
+ GPTQ_MARLIN_MAX_PARALLEL)
506
+
507
+ output = ops.gptq_marlin_gemm(
508
+ a_input,
509
+ marlin_w_q,
510
+ marlin_s,
511
+ marlin_zp,
512
+ g_idx,
513
+ g_idx_sort_indices,
514
+ workspace.scratch,
515
+ quant_type,
516
+ a_input.shape[0],
517
+ b_weight.shape[0],
518
+ a_input.shape[1],
519
+ is_k_full=True,
520
+ has_zp=True,
521
+ use_fp32_reduce=use_fp32_reduce,
522
+ is_zp_float=True,
523
+ )
524
+
525
+ b_flat = b_weight.reshape(-1, group_size)
526
+ zp_flat = zero.reshape(-1, 1)
527
+ s_flat = scale.reshape(-1, 1)
528
+ dequant = (b_flat - zp_flat) * s_flat
529
+
530
+ output_ref = torch.matmul(a_input,
531
+ dequant.reshape(b_weight.shape).transpose(1, 0))
532
+
533
+ torch.cuda.synchronize()
534
+
535
+ max_diff = compute_max_diff(output, output_ref)
536
+
537
+ assert max_diff < 0.04
538
+
539
+
540
@pytest.mark.skipif(not is_quant_method_supported("qqq"),
541
reason="Marlin is not supported on this GPU type.")
542
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
tests/weight_loading/models.txt
+2
-1
@@ -27,4 +27,5 @@ fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
27
marlin, nm-testing/zephyr-beta-7b-marlin-g128, main
28
marlin, robertgshaw2/zephyr-7b-beta-channelwise-marlin, main
29
qqq, HandH1998/QQQ-Llama-3-8b-g128, main
30
-qqq, HandH1998/QQQ-Llama-3-8b, main
\ No newline at end of file
30
+qqq, HandH1998/QQQ-Llama-3-8b, main
31
+hqq, nm-testing/Llama-3.2-1B-Instruct-HQQ, main
\ No newline at end of file
vllm/_custom_ops.py
+5
-3
@@ -343,7 +343,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
343
size_k: torch.SymInt,
344
is_k_full: bool,
345
has_zp: bool = False,
346
- use_fp32_reduce: bool = False) -> torch.Tensor:
346
+ use_fp32_reduce: bool = False,
347
+ is_zp_float: bool = False) -> torch.Tensor:
348
return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
349
350
@register_fake("_C::ggml_dequantize")
@@ -601,11 +602,12 @@ def gptq_marlin_gemm(a: torch.Tensor,
602
size_k: int,
603
is_k_full: bool,
604
has_zp: bool = False,
604
- use_fp32_reduce: bool = False) -> torch.Tensor:
605
+ use_fp32_reduce: bool = False,
606
+ is_zp_float: bool = False) -> torch.Tensor:
607
return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
608
g_idx, perm, workspace, b_q_type.id,
609
size_m, size_n, size_k, is_k_full,
608
- has_zp, use_fp32_reduce)
610
+ has_zp, use_fp32_reduce, is_zp_float)
611
612
613
# fp8 marlin
vllm/model_executor/layers/linear.py
+2
-1
@@ -27,7 +27,8 @@ WEIGHT_LOADER_V2_SUPPORTED = [
27
"AWQLinearMethod", "GPTQMarlinLinearMethod", "Fp8LinearMethod",
28
"MarlinLinearMethod", "QQQLinearMethod", "GPTQMarlin24LinearMethod",
29
"TPUInt8LinearMethod", "GPTQLinearMethod", "FBGEMMFp8LinearMethod",
30
- "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod"
30
+ "ModelOptFp8LinearMethod", "IPEXAWQLinearMethod", "IPEXGPTQLinearMethod",
31
+ "HQQMarlinMethod"
32
]
33
34
vllm/model_executor/layers/quantization/__init__.py
+2
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
21
GPTQMarlinConfig)
22
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
23
GPTQMarlin24Config)
24
+from vllm.model_executor.layers.quantization.hqq_marlin import HQQMarlinConfig
25
from vllm.model_executor.layers.quantization.ipex_quant import IPEXConfig
26
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
27
from vllm.model_executor.layers.quantization.modelopt import ModelOptFp8Config
@@ -48,6 +49,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
49
"compressed-tensors": CompressedTensorsConfig,
50
"bitsandbytes": BitsAndBytesConfig,
51
"qqq": QQQConfig,
52
+ "hqq": HQQMarlinConfig,
53
"experts_int8": ExpertsInt8Config,
54
"neuron_quant": NeuronQuantConfig,
55
"ipex": IPEXConfig,
vllm/model_executor/layers/quantization/hqq_marlin.py
new
+325
@@ -0,0 +1,325 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+
5
+from vllm import _custom_ops as ops
6
+from vllm.logger import init_logger
7
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
8
+ UnquantizedLinearMethod)
9
+from vllm.model_executor.layers.quantization.base_config import (
10
+ QuantizationConfig, QuantizeMethodBase)
11
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13
+ marlin_make_empty_g_idx, marlin_permute_scales)
14
+from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15
+ MarlinWorkspace)
16
+from vllm.model_executor.layers.quantization.utils.quant_utils import gptq_pack
17
+from vllm.model_executor.parameter import (BasevLLMParameter,
18
+ GroupQuantScaleParameter,
19
+ PackedvLLMParameter)
20
+from vllm.scalar_type import scalar_types
21
+
22
+logger = init_logger(__name__)
23
+
24
+
25
+class HQQMarlinConfig(QuantizationConfig):
26
+ """Config class for HQQ Marlin"""
27
+
28
+ def __init__(
29
+ self,
30
+ weight_bits: int,
31
+ group_size: int,
32
+ skip_modules: Optional[List[str]] = None,
33
+ ) -> None:
34
+ assert group_size == 64, ("The only supported HQQ group size is "
35
+ "currently 64.")
36
+ assert weight_bits == 4, ("The only supported HQQ quantization "
37
+ "bitsize is currently 4.")
38
+
39
+ self.weight_bits = weight_bits
40
+ self.group_size = group_size
41
+ self.pack_factor = 32 // weight_bits # packed into int32 in GPTQ format
42
+ self.quant_type = scalar_types.uint4
43
+ self.skip_modules = skip_modules
44
+
45
+ def __repr__(self) -> str:
46
+ return (f"HQQMarlinConfig(quant_type={self.quant_type}, "
47
+ f"group_size={self.group_size})")
48
+
49
+ @classmethod
50
+ def get_name(cls) -> str:
51
+ return "hqq"
52
+
53
+ @classmethod
54
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
55
+ return [torch.half, torch.bfloat16]
56
+
57
+ @classmethod
58
+ def get_min_capability(cls) -> int:
59
+ return 80
60
+
61
+ @classmethod
62
+ def get_config_filenames(cls) -> List[str]:
63
+ return ["quantize_config.json"]
64
+
65
+ @classmethod
66
+ def from_config(cls, config: Dict[str, Any]) -> "HQQMarlinConfig":
67
+ wq_params = (config["quant_config"]["weight_quant_params"])
68
+ weight_bits = cls.get_from_keys(wq_params, ["nbits"])
69
+ group_size = cls.get_from_keys(wq_params, ["group_size"])
70
+ skip_modules = config["skip_modules"]
71
+ return cls(weight_bits, group_size, skip_modules)
72
+
73
+ def is_layer_skipped(self, prefix: str) -> bool:
74
+ # Split the prefix into its dot-separated components
75
+ components = prefix.split('.')
76
+
77
+ # Check if any of the skip modules exactly matches any component
78
+ return self.skip_modules is not None and any(
79
+ module_name in components for module_name in self.skip_modules)
80
+
81
+ def get_quant_method(self, layer: torch.nn.Module,
82
+ prefix: str) -> Optional["QuantizeMethodBase"]:
83
+ if isinstance(layer, LinearBase):
84
+ if self.is_layer_skipped(prefix):
85
+ return UnquantizedLinearMethod()
86
+ return HQQMarlinMethod(self)
87
+ return None
88
+
89
+
90
+# Empty HQQ parameter, will be ignored during loading
91
+class HQQEmptyParameter(BasevLLMParameter):
92
+
93
+ def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
94
+ pass
95
+
96
+ def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
97
+ pass
98
+
99
+ def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
100
+ pass
101
+
102
+
103
+def error_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
104
+ raise ValueError("No loader provided for HQQ parameter!")
105
+
106
+
107
+# HQQ packing creates issues with sharding - therefore, prior to loading, we
108
+# repack to GPTQ. We also reshape the weights to their proper GPTQ shape.
109
+class HQQweightParameter(PackedvLLMParameter):
110
+
111
+ # unpack function from https://github.com/mobiusml/hqq
112
+ def unpack_4bit_u8(self,
113
+ W_q: torch.Tensor) -> torch.Tensor: # uint8/2 > uint8
114
+ assert self.weight_bits == 4, "Unsupported quant bitsize (must be 4)"
115
+
116
+ dtype = torch.uint8
117
+ step = W_q.shape[0]
118
+ tmp = torch.empty([2 * step, W_q.shape[1]],
119
+ dtype=dtype,
120
+ device=W_q.device)
121
+ tmp[:step] = (W_q & 0b11110000) >> 4
122
+ tmp[step:] = W_q & 0b00001111
123
+ return tmp
124
+
125
+ def __init__(self, packed_factor: int, packed_dim: int, weight_bits: int,
126
+ **kwargs):
127
+ super().__init__(packed_factor, packed_dim, None, **kwargs)
128
+ self.weight_bits = weight_bits
129
+ self.input_shape = self.shape[self.input_dim] * self.packed_factor
130
+ self.output_shape = self.shape[self.output_dim]
131
+
132
+ def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
133
+ loaded_weight = self.unpack_4bit_u8(loaded_weight)
134
+ loaded_weight = loaded_weight.reshape(-1, self.input_shape).transpose(
135
+ 1, 0)
136
+ loaded_weight = gptq_pack(loaded_weight, self.weight_bits,
137
+ loaded_weight.shape[0],
138
+ loaded_weight.shape[1])
139
+ super().load_merged_column_weight(loaded_weight, **kwargs)
140
+
141
+ def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
142
+ loaded_weight = self.unpack_4bit_u8(loaded_weight)
143
+ loaded_weight = loaded_weight.reshape(self.output_shape,
144
+ -1).transpose(1, 0)
145
+ loaded_weight = gptq_pack(loaded_weight, self.weight_bits,
146
+ loaded_weight.shape[0],
147
+ loaded_weight.shape[1])
148
+ super().load_row_parallel_weight(loaded_weight)
149
+
150
+ def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
151
+ loaded_weight = self.unpack_4bit_u8(loaded_weight)
152
+ loaded_weight = loaded_weight.reshape(-1, self.input_shape).transpose(
153
+ 1, 0)
154
+ loaded_weight = gptq_pack(loaded_weight, self.weight_bits,
155
+ loaded_weight.shape[0],
156
+ loaded_weight.shape[1])
157
+ super().load_qkv_weight(loaded_weight, **kwargs)
158
+
159
+
160
+# Zero points and scales in HQQ must also be reshaped to correspond to W_q's
161
+# GPTQ shape (transposed - we transpose them too when processing weights).
162
+class HQQZeroScaleParameter(GroupQuantScaleParameter):
163
+
164
+ def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs):
165
+ loaded_weight = loaded_weight.reshape(-1, self.shape[1])
166
+ super().load_merged_column_weight(loaded_weight, **kwargs)
167
+
168
+ def load_row_parallel_weight(self, loaded_weight: torch.Tensor):
169
+ loaded_weight = loaded_weight.reshape(self.shape[0], -1)
170
+ super().load_row_parallel_weight(loaded_weight)
171
+
172
+ def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs):
173
+ loaded_weight = loaded_weight.reshape(-1, self.shape[1])
174
+ super().load_qkv_weight(loaded_weight, **kwargs)
175
+
176
+
177
+class HQQMarlinMethod(LinearMethodBase):
178
+ """Linear method for HQQ Marlin.
179
+ """
180
+
181
+ def __init__(
182
+ self,
183
+ quant_config: HQQMarlinConfig,
184
+ ):
185
+ self.quant_config = quant_config
186
+
187
+ def create_weights(
188
+ self,
189
+ layer: torch.nn.Module,
190
+ input_size_per_partition: int,
191
+ output_partition_sizes: List[int],
192
+ input_size: int,
193
+ output_size: int,
194
+ params_dtype: torch.dtype,
195
+ **extra_weight_attrs,
196
+ ) -> None:
197
+ self.output_size_per_partition = sum(output_partition_sizes)
198
+ self.input_size_per_partition = input_size_per_partition
199
+
200
+ weight_loader = extra_weight_attrs.get("weight_loader", error_loader)
201
+
202
+ self.scales_and_zp_size = (input_size_per_partition //
203
+ self.quant_config.group_size)
204
+
205
+ qweight = HQQweightParameter(
206
+ data=torch.empty(
207
+ self.input_size_per_partition // self.quant_config.pack_factor,
208
+ self.output_size_per_partition,
209
+ dtype=torch.int32,
210
+ ),
211
+ input_dim=0,
212
+ output_dim=1,
213
+ packed_dim=0,
214
+ packed_factor=self.quant_config.pack_factor,
215
+ weight_bits=self.quant_config.weight_bits,
216
+ weight_loader=weight_loader)
217
+
218
+ zeros = HQQZeroScaleParameter(data=torch.empty(
219
+ self.output_size_per_partition,
220
+ self.scales_and_zp_size,
221
+ dtype=params_dtype,
222
+ ),
223
+ input_dim=1,
224
+ output_dim=0,
225
+ weight_loader=weight_loader)
226
+
227
+ scales = HQQZeroScaleParameter(data=torch.empty(
228
+ self.output_size_per_partition,
229
+ self.scales_and_zp_size,
230
+ dtype=params_dtype,
231
+ ),
232
+ input_dim=1,
233
+ output_dim=0,
234
+ weight_loader=weight_loader)
235
+
236
+ layer.register_parameter("W_q", qweight)
237
+ layer.register_parameter("zero", zeros)
238
+ layer.register_parameter("scale", scales)
239
+
240
+ # Ignore extra parameters in the HQQ model.
241
+ # To be added as needed.
242
+ ignore_parameters = ("axis", "channel_wise", "compute_dtype",
243
+ "encoded_state_dict", "group_size", "nbits",
244
+ "offload_meta", "optimize", "packing",
245
+ "quant_scale", "quant_zero", "round_zero",
246
+ "shape", "stores_quant_config",
247
+ "unpack_view_dtype", "view_as_float")
248
+ for name in ignore_parameters:
249
+ layer.register_parameter(
250
+ name,
251
+ HQQEmptyParameter(data=torch.empty(0),
252
+ weight_loader=weight_loader))
253
+
254
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
255
+ dev = layer.W_q.device
256
+
257
+ # Repack to Marlin
258
+ sort_indices = torch.empty(0, dtype=torch.int, device=dev)
259
+ marlin_w_q = ops.gptq_marlin_repack(
260
+ layer.W_q,
261
+ sort_indices,
262
+ self.input_size_per_partition,
263
+ self.output_size_per_partition,
264
+ self.quant_config.weight_bits,
265
+ ).to(dev)
266
+ marlin_s = marlin_permute_scales(layer.scale.transpose(1, 0),
267
+ self.input_size_per_partition,
268
+ self.output_size_per_partition,
269
+ self.quant_config.group_size).to(dev)
270
+ marlin_zp = marlin_permute_scales(layer.zero.transpose(1, 0),
271
+ self.input_size_per_partition,
272
+ self.output_size_per_partition,
273
+ self.quant_config.group_size).to(dev)
274
+
275
+ layer.g_idx = marlin_make_empty_g_idx(dev)
276
+ layer.g_idx_sort_indices = marlin_make_empty_g_idx(dev)
277
+
278
+ layer.marlin_qweight = marlin_w_q
279
+ layer.marlin_zeros = marlin_zp
280
+ layer.marlin_scales = marlin_s
281
+
282
+ def apply(
283
+ self,
284
+ layer: torch.nn.Module,
285
+ x: torch.Tensor,
286
+ bias: Optional[torch.Tensor] = None,
287
+ ) -> torch.Tensor:
288
+ workspace = MarlinWorkspace(self.output_size_per_partition,
289
+ GPTQ_MARLIN_MIN_THREAD_N,
290
+ GPTQ_MARLIN_MAX_PARALLEL)
291
+
292
+ scales = layer.marlin_scales
293
+ zeros = layer.marlin_zeros
294
+ orig_type = x.dtype
295
+
296
+ if orig_type != torch.float16:
297
+ x = x.to(torch.float16)
298
+ scales = scales.to(torch.float16)
299
+ zeros = zeros.to(torch.float16)
300
+
301
+ marlin_out = ops.gptq_marlin_gemm(
302
+ x,
303
+ layer.marlin_qweight,
304
+ scales,
305
+ zeros,
306
+ layer.g_idx,
307
+ layer.g_idx_sort_indices,
308
+ workspace.scratch,
309
+ scalar_types.uint4,
310
+ x.shape[0],
311
+ self.output_size_per_partition,
312
+ self.input_size_per_partition,
313
+ True, # is_k_full
314
+ True, # has_zp
315
+ True, # use 32-bit reduce
316
+ True, # use float zp
317
+ )
318
+
319
+ if orig_type != torch.float16:
320
+ marlin_out = marlin_out.to(orig_type)
321
+
322
+ if bias is not None:
323
+ marlin_out.add_(bias)
324
+
325
+ return marlin_out
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+4
-2
@@ -303,7 +303,8 @@ def apply_gptq_marlin_linear(
303
size_k=input_size_per_partition,
304
is_k_full=is_k_full,
305
has_zp=False,
306
- use_fp32_reduce=use_fp32_reduce)
306
+ use_fp32_reduce=use_fp32_reduce,
307
+ is_zp_float=False)
308
309
if bias is not None:
310
output.add_(bias) # In-place add
@@ -340,7 +341,8 @@ def apply_awq_marlin_linear(
341
size_k=input_size_per_partition,
342
is_k_full=True,
343
has_zp=True,
343
- use_fp32_reduce=use_fp32_reduce)
344
+ use_fp32_reduce=use_fp32_reduce,
345
+ is_zp_float=False)
346
347
if bias is not None:
348
output.add_(bias) # In-place add