@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e288df0632
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fine-tune gptq_marlin configs to be more similar to marlin (#4626)

alexm-nm committed May 8, 2024 at 20:14 UTC e288df0632d5bdde76c20bed8310b46d35b8e5ac
1 file changed +35 -13
csrc/quantization/gptq_marlin/gptq_marlin.cu
+35 -13
@@ -115,7 +115,8 @@ template <int lut> __device__ inline int lop3(int a, int b, int c) {
115 return res;
116 }
117
118 -// Constructs destination register by taking bytes from 2 sources (based on mask)
118 +// Constructs destination register by taking bytes from 2 sources (based on
119 +// mask)
120 template <int start_byte, int mask>
121 __device__ inline uint32_t prmt(uint32_t a) {
122 uint32_t res;
@@ -933,9 +934,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
934 };
935
936 // Since multiple threadblocks may process parts of the same column slice, we
936 - // finally have to globally reduce over the results. As the striped partitioning
937 - // minimizes the number of such reductions and our outputs are usually rather
938 - // small, we perform this reduction serially in L2 cache.
937 + // finally have to globally reduce over the results. As the striped
938 + // partitioning minimizes the number of such reductions and our outputs are
939 + // usually rather small, we perform this reduction serially in L2 cache.
940 auto global_reduce = [&](bool first = false, bool last = false) {
941 // We are very careful here to reduce directly in the output buffer to
942 // maximize L2 cache utilization in this step. To do this, we write out
@@ -1275,13 +1276,22 @@ typedef struct {
1276 thread_config_t tb_cfg;
1277 } exec_config_t;
1278
1278 -thread_config_t thread_configs[] = {
1279 +thread_config_t small_batch_thread_configs[] = {
1280 // Ordered by priority
1281
1282 // thread_k, thread_n, num_threads
1282 - {64, 256, 256}, // Default (max cache usage)
1283 - {64, 128, 128}, // Reduce N, reduce warps
1284 - {128, 64, 128}, // Reduce N more, but increase K
1283 + {128, 128, 256},
1284 + {64, 128, 128},
1285 + {128, 64, 128},
1286 +};
1287 +
1288 +thread_config_t large_batch_thread_configs[] = {
1289 + // Ordered by priority
1290 +
1291 + // thread_k, thread_n, num_threads
1292 + {64, 256, 256},
1293 + {64, 128, 128},
1294 + {128, 64, 128},
1295
1296 };
1297
@@ -1397,11 +1407,21 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1407 int max_shared_mem) {
1408 int max_m_blocks = 4;
1409 while (max_m_blocks > 0) {
1400 - for (auto th_config : thread_configs) {
1401 - if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1402 - num_bits, group_size, has_act_order, is_k_full,
1403 - max_shared_mem)) {
1404 - return exec_config_t{max_m_blocks, th_config};
1410 + if (prob_m <= 16) {
1411 + for (auto th_config : small_batch_thread_configs) {
1412 + if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1413 + num_bits, group_size, has_act_order, is_k_full,
1414 + max_shared_mem)) {
1415 + return exec_config_t{max_m_blocks, th_config};
1416 + }
1417 + }
1418 + } else {
1419 + for (auto th_config : large_batch_thread_configs) {
1420 + if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1421 + num_bits, group_size, has_act_order, is_k_full,
1422 + max_shared_mem)) {
1423 + return exec_config_t{max_m_blocks, th_config};
1424 + }
1425 }
1426 }
1427
@@ -1574,10 +1594,12 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1594 }
1595 CALL_IF(4, 32, 2, 256)
1596 CALL_IF(4, 16, 4, 256)
1597 + CALL_IF(4, 8, 8, 256)
1598 CALL_IF(4, 8, 4, 128)
1599 CALL_IF(4, 4, 8, 128)
1600 CALL_IF(8, 32, 2, 256)
1601 CALL_IF(8, 16, 4, 256)
1602 + CALL_IF(8, 8, 8, 256)
1603 CALL_IF(8, 8, 4, 128)
1604 CALL_IF(8, 4, 8, 128)
1605 else {