@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75acdaa4b6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Increase precision of GPTQ/AWQ Marlin kernel (#6795)

Alexander Matveev committed Jul 27, 2024 at 17:52 UTC 75acdaa4b616c2e95c55a47d3158ceec9c72c503
6 files changed +168 -44
benchmarks/kernels/benchmark_marlin.py
+18 -5
@@ -10,7 +10,7 @@ from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
10 GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
11 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
13 - GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
13 + MARLIN_SUPPORTED_GROUP_SIZES, MARLIN_SUPPORTED_NUM_BITS)
14 from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
15 MarlinWorkspace, marlin_quantize)
16 from vllm.model_executor.layers.quantization.utils.marlin_utils_test_24 import (
@@ -56,6 +56,8 @@ def bench_run(results: List[benchmark.Measurement], model: str,
56 (marlin_24_w_ref, marlin_24_q_w_comp, marlin_24_meta,
57 marlin_24_s) = marlin_24_quantize(b, num_bits, group_size)
58
59 + marlin_zp = torch.empty(0, dtype=torch.int, device=b.device)
60 +
61 # GPTQ quant
62 (w_ref, q_w, s, g_idx,
63 rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
@@ -87,6 +89,7 @@ def bench_run(results: List[benchmark.Measurement], model: str,
89 "marlin_w_ref": marlin_w_ref,
90 "marlin_q_w": marlin_q_w,
91 "marlin_s": marlin_s,
92 + "marlin_zp": marlin_zp,
93 "marlin_g_idx": marlin_g_idx,
94 "marlin_sort_indices": marlin_sort_indices,
95 "marlin_rand_perm": marlin_rand_perm,
@@ -125,11 +128,21 @@ def bench_run(results: List[benchmark.Measurement], model: str,
128 results.append(
129 benchmark.Timer(
130 stmt=
128 - "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full)", # noqa: E501
131 + "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, False)", # noqa: E501
132 + globals=globals,
133 + label=label,
134 + sub_label=sub_label,
135 + description="gptq_marlin_gemm_fp16",
136 + ).blocked_autorange(min_run_time=min_run_time))
137 +
138 + results.append(
139 + benchmark.Timer(
140 + stmt=
141 + "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_zp, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full, False, True)", # noqa: E501
142 globals=globals,
143 label=label,
144 sub_label=sub_label,
132 - description="gptq_marlin_gemm",
145 + description="gptq_marlin_gemm_fp32",
146 ).blocked_autorange(min_run_time=min_run_time))
147
148 if (num_bits in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS
@@ -183,12 +196,12 @@ def main(args):
196 ) > 0 and is_k_full not in args.limit_k_full:
197 continue
198
186 - for num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
199 + for num_bits in MARLIN_SUPPORTED_NUM_BITS:
200 if len(args.limit_num_bits
201 ) > 0 and num_bits not in args.limit_num_bits:
202 continue
203
191 - for group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
204 + for group_size in MARLIN_SUPPORTED_GROUP_SIZES:
205 if len(
206 args.limit_group_size
207 ) > 0 and group_size not in args.limit_group_size:
csrc/ops.h
+2 -1
@@ -93,7 +93,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
93 torch::Tensor& g_idx, torch::Tensor& perm,
94 torch::Tensor& workspace, int64_t num_bits,
95 int64_t size_m, int64_t size_n, int64_t size_k,
96 - bool is_k_full, bool has_zp);
96 + bool is_k_full, bool has_zp,
97 + bool use_fp32_reduce);
98
99 torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
100 int64_t size_k, int64_t size_n,
csrc/quantization/gptq_marlin/gptq_marlin.cu
+122 -28
@@ -59,14 +59,16 @@ __global__ void Marlin(
59 const int4* __restrict__ A, // fp16 input matrix of shape mxk
60 const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
61 int4* __restrict__ C, // fp16 output buffer of shape mxn
62 + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce)
63 const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
64 // (k/groupsize)xn
65 const int* __restrict__ g_idx, // int32 group indices of shape k
65 - int num_groups, // number of scale groups per output channel
66 - int prob_m, // batch dimension m
67 - int prob_n, // output dimension n
68 - int prob_k, // reduction dimension k
69 - int* locks // extra global storage for barrier synchronization
66 + int num_groups, // number of scale groups per output channel
67 + int prob_m, // batch dimension m
68 + int prob_n, // output dimension n
69 + int prob_k, // reduction dimension k
70 + int* locks, // extra global storage for barrier synchronization
71 + bool use_fp32_reduce // whether to use fp32 global reduce
72 ) {}
73
74 } // namespace gptq_marlin
@@ -532,16 +534,18 @@ __global__ void Marlin(
534 const int4* __restrict__ A, // fp16 input matrix of shape mxk
535 const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
536 int4* __restrict__ C, // fp16 output buffer of shape mxn
537 + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce)
538 const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
539 // (k/groupsize)xn
540 const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape
541 // (k/groupsize)x(n/pack_factor)
542 const int* __restrict__ g_idx, // int32 group indices of shape k
540 - int num_groups, // number of scale groups per output channel
541 - int prob_m, // batch dimension m
542 - int prob_n, // output dimension n
543 - int prob_k, // reduction dimension k
544 - int* locks // extra global storage for barrier synchronization
543 + int num_groups, // number of scale groups per output channel
544 + int prob_m, // batch dimension m
545 + int prob_n, // output dimension n
546 + int prob_k, // reduction dimension k
547 + int* locks, // extra global storage for barrier synchronization
548 + bool use_fp32_reduce // whether to use fp32 global reduce
549 ) {
550 // Each threadblock processes one "stripe" of the B matrix with (roughly) the
551 // same size, which might involve multiple column "slices" (of width 16 *
@@ -595,6 +599,8 @@ __global__ void Marlin(
599 int slice_idx; // index of threadblock in current slice; numbered bottom to
600 // top
601
602 + int par_id = 0;
603 +
604 // We can easily implement parallel problem execution by just remapping
605 // indices and advancing global pointers
606 if (slice_col_par >= n_tiles) {
@@ -602,6 +608,7 @@ __global__ void Marlin(
608 C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8;
609 locks += (slice_col_par / n_tiles) * n_tiles;
610 slice_col = slice_col_par % n_tiles;
611 + par_id = slice_col_par / n_tiles;
612 }
613
614 // Compute all information about the current slice which is required for
@@ -632,6 +639,7 @@ __global__ void Marlin(
639 C += 16 * thread_m_blocks * prob_n / 8;
640 locks += n_tiles;
641 slice_col = 0;
642 + par_id++;
643 }
644 };
645 init_slice();
@@ -1321,7 +1329,7 @@ __global__ void Marlin(
1329 // finally have to globally reduce over the results. As the striped
1330 // partitioning minimizes the number of such reductions and our outputs are
1331 // usually rather small, we perform this reduction serially in L2 cache.
1324 - auto global_reduce = [&](bool first = false, bool last = false) {
1332 + auto global_reduce_fp16 = [&](bool first = false, bool last = false) {
1333 // We are very careful here to reduce directly in the output buffer to
1334 // maximize L2 cache utilization in this step. To do this, we write out
1335 // results in FP16 (but still reduce with FP32 compute).
@@ -1382,6 +1390,53 @@ __global__ void Marlin(
1390 }
1391 };
1392
1393 + // Globally reduce over threadblocks that compute the same column block.
1394 + // We use a tmp C buffer to reduce in full fp32 precision.
1395 + auto global_reduce_fp32 = [&](bool first = false, bool last = false) {
1396 + constexpr int tb_m = thread_m_blocks * 16;
1397 + constexpr int tb_n = thread_n_blocks * 16;
1398 +
1399 + constexpr int c_size = tb_m * tb_n * sizeof(float) / 16;
1400 +
1401 + constexpr int active_threads = 32 * thread_n_blocks / 4;
1402 + bool is_th_active = threadIdx.x < active_threads;
1403 +
1404 + int par_offset = c_size * n_tiles * par_id;
1405 + int slice_offset = c_size * slice_col;
1406 +
1407 + constexpr int num_floats = thread_m_blocks * 4 * 2 * 4;
1408 + constexpr int th_size = num_floats * sizeof(float) / 16;
1409 +
1410 + int c_cur_offset = par_offset + slice_offset;
1411 +
1412 + if (!is_th_active) {
1413 + return;
1414 + }
1415 +
1416 + if (!first) {
1417 + float* frag_c_ptr = reinterpret_cast<float*>(&frag_c);
1418 + #pragma unroll
1419 + for (int k = 0; k < th_size; k++) {
1420 + sh[threadIdx.x] =
1421 + C_tmp[c_cur_offset + active_threads * k + threadIdx.x];
1422 +
1423 + float* sh_c_ptr = reinterpret_cast<float*>(&sh[threadIdx.x]);
1424 + #pragma unroll
1425 + for (int f = 0; f < 4; f++) {
1426 + frag_c_ptr[k * 4 + f] += sh_c_ptr[f];
1427 + }
1428 + }
1429 + }
1430 +
1431 + if (!last) {
1432 + int4* frag_c_ptr = reinterpret_cast<int4*>(&frag_c);
1433 + #pragma unroll
1434 + for (int k = 0; k < th_size; k++) {
1435 + C_tmp[c_cur_offset + active_threads * k + threadIdx.x] = frag_c_ptr[k];
1436 + }
1437 + }
1438 + };
1439 +
1440 // Write out the reduce final result in the correct layout. We only actually
1441 // reshuffle matrix fragments in this step, the reduction above is performed
1442 // in fragment layout.
@@ -1606,7 +1661,11 @@ __global__ void Marlin(
1661 if (slice_count > 1) { // only globally reduce if there is more than one
1662 // block in a slice
1663 barrier_acquire(&locks[slice_col], slice_idx);
1609 - global_reduce(slice_idx == 0, last);
1664 + if (use_fp32_reduce) {
1665 + global_reduce_fp32(slice_idx == 0, last);
1666 + } else {
1667 + global_reduce_fp16(slice_idx == 0, last);
1668 + }
1669 barrier_release(&locks[slice_col], last);
1670 }
1671 if (last) // only the last block in a slice actually writes the result
@@ -1661,8 +1720,8 @@ __global__ void Marlin(
1720 THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, HAS_ACT_ORDER, \
1721 HAS_ZP, GROUP_BLOCKS> \
1722 <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1664 - A_ptr, B_ptr, C_ptr, s_ptr, zp_ptr, g_idx_ptr, num_groups, \
1665 - prob_m, prob_n, prob_k, locks); \
1723 + A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, zp_ptr, g_idx_ptr, \
1724 + num_groups, prob_m, prob_n, prob_k, locks, use_fp32_reduce); \
1725 }
1726
1727 typedef struct {
@@ -1801,6 +1860,27 @@ bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,
1860 return true;
1861 }
1862
1863 +int determine_reduce_max_m(int prob_m, int max_par) {
1864 + constexpr int tile_m_size = 16;
1865 +
1866 + if (prob_m <= tile_m_size) {
1867 + return tile_m_size;
1868 +
1869 + } else if (prob_m <= tile_m_size * 2) {
1870 + return tile_m_size * 2;
1871 +
1872 + } else if (prob_m <= tile_m_size * 3) {
1873 + return tile_m_size * 3;
1874 +
1875 + } else if (prob_m <= tile_m_size * 4) {
1876 + return tile_m_size * 4;
1877 +
1878 + } else {
1879 + int cur_par = min(div_ceil(prob_m, tile_m_size * 4), max_par);
1880 + return tile_m_size * 4 * cur_par;
1881 + }
1882 +}
1883 +
1884 exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1885 int num_bits, int group_size,
1886 bool has_act_order, bool is_k_full,
@@ -1880,13 +1960,13 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1960 __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, false, true, 8, NUM_THREADS)
1961
1962 template <typename scalar_t>
1883 -void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, void* zp,
1884 - void* g_idx, void* perm, void* a_tmp, int prob_m,
1885 - int prob_n, int prob_k, void* workspace, int num_bits,
1886 - bool has_act_order, bool is_k_full, bool has_zp,
1887 - int num_groups, int group_size, int dev,
1963 +void marlin_mm_f16i4(const void* A, const void* B, void* C, void* C_tmp,
1964 + void* s, void* zp, void* g_idx, void* perm, void* a_tmp,
1965 + int prob_m, int prob_n, int prob_k, void* workspace,
1966 + int num_bits, bool has_act_order, bool is_k_full,
1967 + bool has_zp, int num_groups, int group_size, int dev,
1968 cudaStream_t stream, int thread_k, int thread_n, int sms,
1889 - int max_par) {
1969 + int max_par, bool use_fp32_reduce) {
1970 TORCH_CHECK(num_bits == 4 || num_bits == 8,
1971 "num_bits must be 4 or 8. Got = ", num_bits);
1972 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
@@ -1970,6 +2050,7 @@ void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, void* zp,
2050 const int4* A_ptr = (const int4*)A;
2051 const int4* B_ptr = (const int4*)B;
2052 int4* C_ptr = (int4*)C;
2053 + int4* C_tmp_ptr = (int4*)C_tmp;
2054 const int4* s_ptr = (const int4*)s;
2055 const int4* zp_ptr = (const int4*)zp;
2056 const int* g_idx_ptr = (const int*)g_idx;
@@ -2049,7 +2130,8 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2130 torch::Tensor& g_idx, torch::Tensor& perm,
2131 torch::Tensor& workspace, int64_t num_bits,
2132 int64_t size_m, int64_t size_n, int64_t size_k,
2052 - bool is_k_full, bool has_zp) {
2133 + bool is_k_full, bool has_zp,
2134 + bool use_fp32_reduce) {
2135 // Verify num_bits
2136 TORCH_CHECK(num_bits == 4 || num_bits == 8,
2137 "num_bits must be 4 or 8. Got = ", num_bits);
@@ -2099,6 +2181,17 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2181 torch::Tensor c = torch::empty({size_m, size_n}, options);
2182 torch::Tensor a_tmp = torch::empty({size_m, size_k}, options);
2183
2184 + // Alloc C tmp buffer that is going to be used for the global reduce
2185 + int reduce_max_m = marlin::determine_reduce_max_m(size_m, marlin::max_par);
2186 + int reduce_n = size_n;
2187 + auto options_fp32 =
2188 + torch::TensorOptions().dtype(at::kFloat).device(a.device());
2189 + if (!use_fp32_reduce) {
2190 + reduce_max_m = 0;
2191 + reduce_n = 0;
2192 + }
2193 + torch::Tensor c_tmp = torch::empty({reduce_max_m, reduce_n}, options_fp32);
2194 +
2195 // thread_k: `k` size of a thread_tile in `weights` (can usually be left as
2196 // auto -1)
2197 int thread_k = -1;
@@ -2171,20 +2264,21 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2264 if (a.scalar_type() == at::ScalarType::Half) {
2265 marlin::marlin_mm_f16i4<half>(
2266 a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
2174 - b_scales.data_ptr<at::Half>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2175 - perm.data_ptr(), a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2267 + c_tmp.data_ptr<float>(), b_scales.data_ptr<at::Half>(),
2268 + b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2269 + a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2270 workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2271 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2178 - thread_k, thread_n, sms, marlin::max_par);
2272 + thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2273 } else if (a.scalar_type() == at::ScalarType::BFloat16) {
2274 marlin::marlin_mm_f16i4<nv_bfloat16>(
2275 a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
2182 - c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(),
2183 - b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2184 - a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2276 + c.data_ptr<at::BFloat16>(), c_tmp.data_ptr<float>(),
2277 + b_scales.data_ptr<at::BFloat16>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2278 + perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2279 workspace.data_ptr(), num_bits, has_act_order, is_k_full, has_zp,
2280 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2187 - thread_k, thread_n, sms, marlin::max_par);
2281 + thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2282 } else {
2283 TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16");
2284 }
tests/kernels/test_marlin_gemm.py
+10 -3
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
27
28 ACT_ORDER_OPTS = [False, True]
29 K_FULL_OPTS = [False, True]
30 +USE_FP32_REDUCE_OPTS = [False, True]
31
32 MARLIN_K_CHUNKS = [128]
33 MARLIN_N_CHUNKS = [64, 128, 256]
@@ -175,6 +176,7 @@ def test_awq_marlin_repack(k_chunk, n_chunk, num_bits, group_size,
176 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
177 @pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
178 @pytest.mark.parametrize("is_k_full", K_FULL_OPTS)
179 +@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
180 def test_gptq_marlin_gemm(
181 k_chunk,
182 n_chunk,
@@ -183,6 +185,7 @@ def test_gptq_marlin_gemm(
185 mnk_factors,
186 act_order,
187 is_k_full,
188 + use_fp32_reduce,
189 ):
190 m_factor, n_factor, k_factor = mnk_factors
191
@@ -222,8 +225,9 @@ def test_gptq_marlin_gemm(
225 a_input.shape[0],
226 b_weight.shape[1],
227 a_input.shape[1],
225 - is_k_full,
228 + is_k_full=is_k_full,
229 has_zp=False,
230 + use_fp32_reduce=use_fp32_reduce,
231 )
232 output_ref = torch.matmul(a_input, w_ref)
233
@@ -365,12 +369,14 @@ def test_fp8_marlin_gemm(
369 @pytest.mark.parametrize("num_bits", MARLIN_SUPPORTED_NUM_BITS)
370 @pytest.mark.parametrize("group_size", MARLIN_SUPPORTED_GROUP_SIZES)
371 @pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
372 +@pytest.mark.parametrize("use_fp32_reduce", USE_FP32_REDUCE_OPTS)
373 def test_awq_marlin_gemm(
374 k_chunk,
375 n_chunk,
376 num_bits,
377 group_size,
378 mnk_factors,
379 + use_fp32_reduce,
380 ):
381 m_factor, n_factor, k_factor = mnk_factors
382
@@ -407,8 +413,9 @@ def test_awq_marlin_gemm(
413 a_input.shape[0],
414 b_weight.shape[1],
415 a_input.shape[1],
410 - is_k_full,
411 - has_zp,
416 + is_k_full=is_k_full,
417 + has_zp=has_zp,
418 + use_fp32_reduce=use_fp32_reduce,
419 )
420 output_ref = torch.matmul(a_input, w_ref)
421
vllm/_custom_ops.py
+3 -3
@@ -286,12 +286,12 @@ def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
286 b_scales: torch.Tensor, b_zeros: torch.Tensor,
287 g_idx: torch.Tensor, perm: torch.Tensor,
288 workspace: torch.Tensor, num_bits: int, size_m: int,
289 - size_n: int, size_k: int, is_k_full: bool,
290 - has_zp: bool) -> torch.Tensor:
289 + size_n: int, size_k: int, is_k_full: bool, has_zp: bool,
290 + use_fp32_reduce: bool) -> torch.Tensor:
291 return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
292 g_idx, perm, workspace, num_bits,
293 size_m, size_n, size_k, is_k_full,
294 - has_zp)
294 + has_zp, use_fp32_reduce)
295
296
297 # fp8 marlin
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+13 -4
@@ -16,6 +16,11 @@ GPTQ_MARLIN_MAX_PARALLEL = 16
16 MARLIN_SUPPORTED_NUM_BITS = [4, 8]
17 MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
18
19 +# In case there is a performance issue with Marlin, the variable below can be
20 +# changed to False, which allows Marlin to perform global reductions in fp16
21 +# precision (instead of fp32), and therefore, save on some memory movements.
22 +USE_FP32_REDUCE_DEFAULT = True
23 +
24
25 def _check_marlin_supported(num_bits: int, group_size: int, is_sym: bool,
26 min_capability: Optional[int],
@@ -244,7 +249,8 @@ def apply_gptq_marlin_linear(
249 output_size_per_partition: int,
250 input_size_per_partition: int,
251 is_k_full: bool,
247 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
252 + bias: Optional[torch.Tensor] = None,
253 + use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
254 reshaped_x = input.reshape(-1, input.shape[-1])
255 out_shape = input.shape[:-1] + (output_size_per_partition, )
256
@@ -260,7 +266,8 @@ def apply_gptq_marlin_linear(
266 size_n=output_size_per_partition,
267 size_k=input_size_per_partition,
268 is_k_full=is_k_full,
263 - has_zp=False)
269 + has_zp=False,
270 + use_fp32_reduce=use_fp32_reduce)
271
272 if bias is not None:
273 output.add_(bias) # In-place add
@@ -279,7 +286,8 @@ def apply_awq_marlin_linear(
286 num_bits: int,
287 output_size_per_partition: int,
288 input_size_per_partition: int,
282 - bias: Optional[torch.Tensor] = None) -> torch.Tensor:
289 + bias: Optional[torch.Tensor] = None,
290 + use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
291 reshaped_x = input.reshape(-1, input.shape[-1])
292 out_shape = input.shape[:-1] + (output_size_per_partition, )
293
@@ -295,7 +303,8 @@ def apply_awq_marlin_linear(
303 size_n=output_size_per_partition,
304 size_k=input_size_per_partition,
305 is_k_full=True,
298 - has_zp=True)
306 + has_zp=True,
307 + use_fp32_reduce=use_fp32_reduce)
308
309 if bias is not None:
310 output.add_(bias) # In-place add