A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Increase precision of GPTQ/AWQ Marlin kernel (#6795)
Alexander Matveev committed
Jul 27, 2024 at 17:52 UTC
75acdaa4b616c2e95c55a47d3158ceec9c72c503
6 files changed
+168
-44
benchmarks/kernels/benchmark_marlin.py
+18
-5
@@ -10,7 +10,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
10
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
11
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12
GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13
- GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
13
+ MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS)
14
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15
MarlinWorkspace, marlin_quantize)
16
from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
@@ -56,6 +56,8 @@ def bench_run(results: List[benchmark.Measurement], model: str,
56
(marlin_24_w_ref, marlin_24_q_w_comp, marlin_24_meta,
57
marlin_24_s) = marlin_24_quantize(b, num_bits, group_size)
58
59
+ marlin_zp = torch.empty(0, dtype=torch.int, device=b.device)
60
+
61
# GPTQ quant
62
(w_ref, q_w, s, g_idx,
63
rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
@@ -87,6 +89,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
89
"marlin_w_ref": marlin_w_ref,
90
"marlin_q_w": marlin_q_w,
91
"marlin_s": marlin_s,
92
+ "marlin_zp": marlin_zp,
93
"marlin_g_idx": marlin_g_idx,
94
"marlin_sort_indices": marlin_sort_indices,
95
"marlin_rand_perm": marlin_rand_perm,
@@ -125,11 +128,21 @@ def bench_run(results: List[benchmark.Measurement], model: str,
128
results.append(
129
benchmark.Timer(
130
stmt=
128
- "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full)", # noqa: E501
131
+ "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
132
+ globals=globals,
133
+ label=label,
134
+ sub_label=sub_label,
135
+ description="gptq_marlin_gemm_fp16",
136
+ ).blocked_autorange(min_run_time=min_run_time))
137
+
138
+ results.append(
139
+ benchmark.Timer(
140
+ stmt=
141
+ "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
142
globals=globals,
143
label=label,
144
sub_label=sub_label,
132
- description="gptq_marlin_gemm",
145
+ description="gptq_marlin_gemm_fp32",
146
).blocked_autorange(min_run_time=min_run_time))
147
148
if (num_bits in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS
@@ -183,12 +196,12 @@ def main(args):
196
) > 0 and is_k_full not in args.limit_k_full:
197
continue
198
186
- for num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
199
+ for num_bits in MARLIN_SUPPORTED_NUM_BITS:
200
if len(args.limit_num_bits
201
) > 0 and num_bits not in args.limit_num_bits:
202
continue
203
191
- for group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
204
+ for group_size in MARLIN_SUPPORTED_GROUP_SIZES:
205
if len(
206
args.limit_group_size
207
) > 0 and group_size not in args.limit_group_size:
csrc/ops.h
+2
-1
@@ -93,7 +93,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
93
torch::Tensor& g_idx, torch::Tensor& perm,
94
torch::Tensor& workspace, int64_t num_bits,
95
int64_t size_m, int64_t size_n, int64_t size_k,
96
- bool is_k_full, bool has_zp);
96
+ bool is_k_full, bool has_zp,
97
+ bool use_fp32_reduce);
98
99
torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
100
int64_t size_k, int64_t size_n,
csrc/quantization/gptq_marlin/gptq_marlin.cu
+122
-28
@@ -59,14 +59,16 @@ __global__ void Marlin(
59
const int4* __restrict__ A, // fp16 input matrix of shape mxk
60
const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
61
int4* __restrict__ C, // fp16 output buffer of shape mxn
62
+ int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce)
63
const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
64
// (k/groupsize)xn
65
const int* __restrict__ g_idx, // int32 group indices of shape k
65
- int num_groups, // number of scale groups per output channel
66
- int prob_m, // batch dimension m
67
- int prob_n, // output dimension n
68
- int prob_k, // reduction dimension k
69
- int* locks // extra global storage for barrier synchronization
66
+ int num_groups, // number of scale groups per output channel
67
+ int prob_m, // batch dimension m
68
+ int prob_n, // output dimension n
69
+ int prob_k, // reduction dimension k
70
+ int* locks, // extra global storage for barrier synchronization
71
+ bool use_fp32_reduce // whether to use fp32 global reduce
72
) {}
73
74
} // namespace gptq_marlin
@@ -532,16 +534,18 @@ __global__ void Marlin(
534
const int4* __restrict__ A, // fp16 input matrix of shape mxk
535
const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
536
int4* __restrict__ C, // fp16 output buffer of shape mxn
537
+ int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce)
538
const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
539
// (k/groupsize)xn
540
const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
541
// (k/groupsize)x(n/pack_factor)
542
const int* __restrict__ g_idx, // int32 group indices of shape k
540
- int num_groups, // number of scale groups per output channel
541
- int prob_m, // batch dimension m
542
- int prob_n, // output dimension n
543
- int prob_k, // reduction dimension k
544
- int* locks // extra global storage for barrier synchronization
543
+ int num_groups, // number of scale groups per output channel
544
+ int prob_m, // batch dimension m
545
+ int prob_n, // output dimension n
546
+ int prob_k, // reduction dimension k
547
+ int* locks, // extra global storage for barrier synchronization
548
+ bool use_fp32_reduce // whether to use fp32 global reduce
549
) {
550
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
551
// same size, which might involve multiple column "slices" (of width 16 *
@@ -595,6 +599,8 @@ __global__ void Marlin(
599
int slice_idx; // index of threadblock in current slice; numbered bottom to
600
// top
601
602
+ int par_id = 0;
603
+
604
// We can easily implement parallel problem execution by just remapping
605
// indices and advancing global pointers
606
if (slice_col_par >= n_tiles) {
@@ -602,6 +608,7 @@ __global__ void Marlin(
608
C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8;
609
locks += (slice_col_par / n_tiles) * n_tiles;
610
slice_col = slice_col_par % n_tiles;
611
+ par_id = slice_col_par / n_tiles;
612
}
613
614
// Compute all information about the current slice which is required for
@@ -632,6 +639,7 @@ __global__ void Marlin(
639
C += 16 * thread_m_blocks * prob_n / 8;
640
locks += n_tiles;
641
slice_col = 0;
642
+ par_id++;
643
}
644
};
645
init_slice();
@@ -1321,7 +1329,7 @@ __global__ void Marlin(
1329
// finally have to globally reduce over the results. As the striped
1330
// partitioning minimizes the number of such reductions and our outputs are
1331
// usually rather small, we perform this reduction serially in L2 cache.
1324
- auto global_reduce = [&](bool first = false, bool last = false) {
1332
+ auto global_reduce_fp16 = [&](bool first = false, bool last = false) {
1333
// We are very careful here to reduce directly in the output buffer to
1334
// maximize L2 cache utilization in this step. To do this, we write out
1335
// results in FP16 (but still reduce with FP32 compute).
@@ -1382,6 +1390,53 @@ __global__ void Marlin(
1390
}
1391
};
1392
1393
+ // Globally reduce over threadblocks that compute the same column block.
1394
+ // We use a tmp C buffer to reduce in full fp32 precision.
1395
+ auto global_reduce_fp32 = [&](bool first = false, bool last = false) {
1396
+ constexpr int tb_m = thread_m_blocks * 16;
1397
+ constexpr int tb_n = thread_n_blocks * 16;
1398
+
1399
+ constexpr int c_size = tb_m * tb_n * sizeof(float) / 16;
1400
+
1401
+ constexpr int active_threads = 32 * thread_n_blocks / 4;
1402
+ bool is_th_active = threadIdx.x < active_threads;
1403
+
1404
+ int par_offset = c_size * n_tiles * par_id;
1405
+ int slice_offset = c_size * slice_col;
1406
+
1407
+ constexpr int num_floats = thread_m_blocks * 4 * 2 * 4;
1408
+ constexpr int th_size = num_floats * sizeof(float) / 16;
1409
+
1410
+ int c_cur_offset = par_offset + slice_offset;
1411
+
1412
+ if (!is_th_active) {
1413
+ return;
1414
+ }
1415
+
1416
+ if (!first) {
1417
+ float* frag_c_ptr = reinterpret_cast<float*>(&frag_c);
1418
+ #pragma unroll
1419
+ for (int k = 0; k < th_size; k++) {
1420
+ sh[threadIdx.x] =
1421
+ C_tmp[c_cur_offset + active_threads * k + threadIdx.x];
1422
+
1423
+ float* sh_c_ptr = reinterpret_cast<float*>(&sh[threadIdx.x]);
1424
+ #pragma unroll
1425
+ for (int f = 0; f < 4; f++) {
1426
+ frag_c_ptr[k * 4 + f] += sh_c_ptr[f];
1427
+ }
1428
+ }
1429
+ }
1430
+
1431
+ if (!last) {
1432
+ int4* frag_c_ptr = reinterpret_cast<int4*>(&frag_c);
1433
+ #pragma unroll
1434
+ for (int k = 0; k < th_size; k++) {
1435
+ C_tmp[c_cur_offset + active_threads * k + threadIdx.x] = frag_c_ptr[k];
1436
+ }
1437
+ }
1438
+ };
1439
+
1440
// Write out the reduce final result in the correct layout. We only actually
1441
// reshuffle matrix fragments in this step, the reduction above is performed
1442
// in fragment layout.
@@ -1606,7 +1661,11 @@ __global__ void Marlin(
1661
if (slice_count > 1) { // only globally reduce if there is more than one
1662
// block in a slice
1663
barrier_acquire(&locks[slice_col], slice_idx);
1609
- global_reduce(slice_idx == 0, last);
1664
+ if (use_fp32_reduce) {
1665
+ global_reduce_fp32(slice_idx == 0, last);
1666
+ } else {
1667
+ global_reduce_fp16(slice_idx == 0, last);
1668
+ }
1669
barrier_release(&locks[slice_col], last);
1670
}
1671
if (last) // only the last block in a slice actually writes the result
@@ -1661,8 +1720,8 @@ __global__ void Marlin(
1720
THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1721
HAS_ZP, GROUP_BLOCKS> \
1722
<<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1664
- A_ptr, B_ptr, C_ptr, s_ptr, zp_ptr, g_idx_ptr, num_groups, \
1665
- prob_m, prob_n, prob_k, locks); \
1723
+ A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, zp_ptr, g_idx_ptr, \
1724
+ num_groups, prob_m, prob_n, prob_k, locks, use_fp32_reduce); \
1725
}
1726
1727
typedef struct {
@@ -1801,6 +1860,27 @@ bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,
1860
return true;
1861
}
1862
1863
+int determine_reduce_max_m(int prob_m, int max_par) {
1864
+ constexpr int tile_m_size = 16;
1865
+
1866
+ if (prob_m <= tile_m_size) {
1867
+ return tile_m_size;
1868
+
1869
+ } else if (prob_m <= tile_m_size * 2) {
1870
+ return tile_m_size * 2;
1871
+
1872
+ } else if (prob_m <= tile_m_size * 3) {
1873
+ return tile_m_size * 3;
1874
+
1875
+ } else if (prob_m <= tile_m_size * 4) {
1876
+ return tile_m_size * 4;
1877
+
1878
+ } else {
1879
+ int cur_par = min(div_ceil(prob_m, tile_m_size * 4), max_par);
1880
+ return tile_m_size * 4 * cur_par;
1881
+ }
1882
+}
1883
+
1884
exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1885
int num_bits, int group_size,
1886
bool has_act_order, bool is_k_full,
@@ -1880,13 +1960,13 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1960
__CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1961
1962
template <typename scalar_t>
1883
-void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, void* zp,
1884
- void* g_idx, void* perm, void* a_tmp, int prob_m,
1885
- int prob_n, int prob_k, void* workspace, int num_bits,
1886
- bool has_act_order, bool is_k_full, bool has_zp,
1887
- int num_groups, int group_size, int dev,
1963
+void marlin_mm_f16i4(const void* A, const void* B, void* C, void* C_tmp,
1964
+ void* s, void* zp, void* g_idx, void* perm, void* a_tmp,
1965
+ int prob_m, int prob_n, int prob_k, void* workspace,
1966
+ int num_bits, bool has_act_order, bool is_k_full,
1967
+ bool has_zp, int num_groups, int group_size, int dev,
1968
cudaStream_t stream, int thread_k, int thread_n, int sms,
1889
- int max_par) {
1969
+ int max_par, bool use_fp32_reduce) {
1970
TORCH_CHECK(num_bits == 4 || num_bits == 8,
1971
"num_bits must be 4 or 8. Got = ", num_bits);
1972
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
@@ -1970,6 +2050,7 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, void* zp,
2050
const int4* A_ptr = (const int4*)A;
2051
const int4* B_ptr = (const int4*)B;
2052
int4* C_ptr = (int4*)C;
2053
+ int4* C_tmp_ptr = (int4*)C_tmp;
2054
const int4* s_ptr = (const int4*)s;
2055
const int4* zp_ptr = (const int4*)zp;
2056
const int* g_idx_ptr = (const int*)g_idx;
@@ -2049,7 +2130,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2130
torch::Tensor& g_idx, torch::Tensor& perm,
2131
torch::Tensor& workspace, int64_t num_bits,
2132
int64_t size_m, int64_t size_n, int64_t size_k,
2052
- bool is_k_full, bool has_zp) {
2133
+ bool is_k_full, bool has_zp,
2134
+ bool use_fp32_reduce) {
2135
// Verify num_bits
2136
TORCH_CHECK(num_bits == 4 || num_bits == 8,
2137
"num_bits must be 4 or 8. Got = ", num_bits);
@@ -2099,6 +2181,17 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2181
torch::Tensor c = torch::empty({size_m, size_n}, options);
2182
torch::Tensor a_tmp = torch::empty({size_m, size_k}, options);
2183
2184
+ // Alloc C tmp buffer that is going to be used for the global reduce
2185
+ int reduce_max_m = marlin::determine_reduce_max_m(size_m, marlin::max_par);
2186
+ int reduce_n = size_n;
2187
+ auto options_fp32 =
2188
+ torch::TensorOptions().dtype(at::kFloat).device(a.device());
2189
+ if (!use_fp32_reduce) {
2190
+ reduce_max_m = 0;
2191
+ reduce_n = 0;
2192
+ }
2193
+ torch::Tensor c_tmp = torch::empty({reduce_max_m, reduce_n}, options_fp32);
2194
+
2195
// thread_k: `k` size of a thread_tile in `weights` (can usually be left as
2196
// auto -1)
2197
int thread_k = -1;
@@ -2171,20 +2264,21 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2264
if (a.scalar_type() == at::ScalarType::Half) {
2265
marlin::marlin_mm_f16i4<half>(
2266
a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
2174
- b_scales.data_ptr<at::Half>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2175
- perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2267
+ c_tmp.data_ptr<float>(), b_scales.data_ptr<at::Half>(),
2268
+ b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2269
+ a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2270
workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2271
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2178
- thread_k, thread_n, sms, marlin::max_par);
2272
+ thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2273
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
2274
marlin::marlin_mm_f16i4<nv_bfloat16>(
2275
a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
2182
- c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
2183
- b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2184
- a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2276
+ c.data_ptr<at::BFloat16>(), c_tmp.data_ptr<float>(),
2277
+ b_scales.data_ptr<at::BFloat16>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2278
+ perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2279
workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2280
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2187
- thread_k, thread_n, sms, marlin::max_par);
2281
+ thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2282
} else {
2283
TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
2284
}
tests/kernels/test_marlin_gemm.py
+10
-3
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
27
28
ACT_ORDER_OPTS = [False, True]
29
K_FULL_OPTS = [False, True]
30
+USE_FP32_REDUCE_OPTS = [False, True]
31
32
MARLIN_K_CHUNKS = [128]
33
MARLIN_N_CHUNKS = [64, 128, 256]
@@ -175,6 +176,7 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
176
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
177
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
178
@pytest.mark.parametrize("is_k_full", K_FULL_OPTS)
179
+@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
180
def test_gptq_marlin_gemm(
181
k_chunk,
182
n_chunk,
@@ -183,6 +185,7 @@ def test_gptq_marlin_gemm(
185
mnk_factors,
186
act_order,
187
is_k_full,
188
+ use_fp32_reduce,
189
):
190
m_factor, n_factor, k_factor = mnk_factors
191
@@ -222,8 +225,9 @@ def test_gptq_marlin_gemm(
225
a_input.shape[0],
226
b_weight.shape[1],
227
a_input.shape[1],
225
- is_k_full,
228
+ is_k_full=is_k_full,
229
has_zp=False,
230
+ use_fp32_reduce=use_fp32_reduce,
231
)
232
output_ref = torch.matmul(a_input, w_ref)
233
@@ -365,12 +369,14 @@ def test_fp8_marlin_gemm(
369
@pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
370
@pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
371
@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
372
+@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
373
def test_awq_marlin_gemm(
374
k_chunk,
375
n_chunk,
376
num_bits,
377
group_size,
378
mnk_factors,
379
+ use_fp32_reduce,
380
):
381
m_factor, n_factor, k_factor = mnk_factors
382
@@ -407,8 +413,9 @@ def test_awq_marlin_gemm(
413
a_input.shape[0],
414
b_weight.shape[1],
415
a_input.shape[1],
410
- is_k_full,
411
- has_zp,
416
+ is_k_full=is_k_full,
417
+ has_zp=has_zp,
418
+ use_fp32_reduce=use_fp32_reduce,
419
)
420
output_ref = torch.matmul(a_input, w_ref)
421
vllm/_custom_ops.py
+3
-3
@@ -286,12 +286,12 @@ def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
286
b_scales: torch.Tensor, b_zeros: torch.Tensor,
287
g_idx: torch.Tensor, perm: torch.Tensor,
288
workspace: torch.Tensor, num_bits: int, size_m: int,
289
- size_n: int, size_k: int, is_k_full: bool,
290
- has_zp: bool) -> torch.Tensor:
289
+ size_n: int, size_k: int, is_k_full: bool, has_zp: bool,
290
+ use_fp32_reduce: bool) -> torch.Tensor:
291
return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
292
g_idx, perm, workspace, num_bits,
293
size_m, size_n, size_k, is_k_full,
294
- has_zp)
294
+ has_zp, use_fp32_reduce)
295
296
297
# fp8 marlin
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+13
-4
@@ -16,6 +16,11 @@ GPTQ_MARLIN_MAX_PARALLEL = 16
16
MARLIN_SUPPORTED_NUM_BITS = [4, 8]
17
MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
18
19
+# In case there is a performance issue with Marlin, the variable below can be
20
+# changed to False, which allows Marlin to perform global reductions in fp16
21
+# precision (instead of fp32), and therefore, save on some memory movements.
22
+USE_FP32_REDUCE_DEFAULT = True
23
+
24
25
def _check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
26
min_capability: Optional[int],
@@ -244,7 +249,8 @@ def apply_gptq_marlin_linear(
249
output_size_per_partition: int,
250
input_size_per_partition: int,
251
is_k_full: bool,
247
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
252
+ bias: Optional[torch.Tensor] = None,
253
+ use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
254
reshaped_x = input.reshape(-1, input.shape[-1])
255
out_shape = input.shape[:-1] + (output_size_per_partition, )
256
@@ -260,7 +266,8 @@ def apply_gptq_marlin_linear(
266
size_n=output_size_per_partition,
267
size_k=input_size_per_partition,
268
is_k_full=is_k_full,
263
- has_zp=False)
269
+ has_zp=False,
270
+ use_fp32_reduce=use_fp32_reduce)
271
272
if bias is not None:
273
output.add_(bias) # In-place add
@@ -279,7 +286,8 @@ def apply_awq_marlin_linear(
286
num_bits: int,
287
output_size_per_partition: int,
288
input_size_per_partition: int,
282
- bias: Optional[torch.Tensor] = None) -> torch.Tensor:
289
+ bias: Optional[torch.Tensor] = None,
290
+ use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
291
reshaped_x = input.reshape(-1, input.shape[-1])
292
out_shape = input.shape[:-1] + (output_size_per_partition, )
293
@@ -295,7 +303,8 @@ def apply_awq_marlin_linear(
303
size_n=output_size_per_partition,
304
size_k=input_size_per_partition,
305
is_k_full=True,
298
- has_zp=True)
306
+ has_zp=True,
307
+ use_fp32_reduce=use_fp32_reduce)
308
309
if bias is not None:
310
output.add_(bias) # In-place add