A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fine-tune gptq_marlin configs to be more similar to marlin (#4626)
alexm-nm committed
May 8, 2024 at 20:14 UTC
e288df0632d5bdde76c20bed8310b46d35b8e5ac
1 file changed
+35
-13
csrc/quantization/gptq_marlin/gptq_marlin.cu
+35
-13
@@ -115,7 +115,8 @@ template <int lut> __device__ inline int lop3(int a, int b, int c) {
115
return res;
116
}
117
118
-// Constructs destination register by taking bytes from 2 sources (based on mask)
118
+// Constructs destination register by taking bytes from 2 sources (based on
119
+// mask)
120
template <int start_byte, int mask>
121
__device__ inline uint32_t prmt(uint32_t a) {
122
uint32_t res;
@@ -933,9 +934,9 @@ Marlin(const int4 *__restrict__ A, // fp16 input matrix of shape mxk
934
};
935
936
// Since multiple threadblocks may process parts of the same column slice, we
936
- // finally have to globally reduce over the results. As the striped partitioning
937
- // minimizes the number of such reductions and our outputs are usually rather
938
- // small, we perform this reduction serially in L2 cache.
937
+ // finally have to globally reduce over the results. As the striped
938
+ // partitioning minimizes the number of such reductions and our outputs are
939
+ // usually rather small, we perform this reduction serially in L2 cache.
940
auto global_reduce = [&](bool first = false, bool last = false) {
941
// We are very careful here to reduce directly in the output buffer to
942
// maximize L2 cache utilization in this step. To do this, we write out
@@ -1275,13 +1276,22 @@ typedef struct {
1276
thread_config_t tb_cfg;
1277
} exec_config_t;
1278
1278
-thread_config_t thread_configs[] = {
1279
+thread_config_t small_batch_thread_configs[] = {
1280
// Ordered by priority
1281
1282
// thread_k, thread_n, num_threads
1282
- {64, 256, 256}, // Default (max cache usage)
1283
- {64, 128, 128}, // Reduce N, reduce warps
1284
- {128, 64, 128}, // Reduce N more, but increase K
1283
+ {128, 128, 256},
1284
+ {64, 128, 128},
1285
+ {128, 64, 128},
1286
+};
1287
+
1288
+thread_config_t large_batch_thread_configs[] = {
1289
+ // Ordered by priority
1290
+
1291
+ // thread_k, thread_n, num_threads
1292
+ {64, 256, 256},
1293
+ {64, 128, 128},
1294
+ {128, 64, 128},
1295
1296
};
1297
@@ -1397,11 +1407,21 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1407
int max_shared_mem) {
1408
int max_m_blocks = 4;
1409
while (max_m_blocks > 0) {
1400
- for (auto th_config : thread_configs) {
1401
- if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1402
- num_bits, group_size, has_act_order, is_k_full,
1403
- max_shared_mem)) {
1404
- return exec_config_t{max_m_blocks, th_config};
1410
+ if (prob_m <= 16) {
1411
+ for (auto th_config : small_batch_thread_configs) {
1412
+ if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1413
+ num_bits, group_size, has_act_order, is_k_full,
1414
+ max_shared_mem)) {
1415
+ return exec_config_t{max_m_blocks, th_config};
1416
+ }
1417
+ }
1418
+ } else {
1419
+ for (auto th_config : large_batch_thread_configs) {
1420
+ if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1421
+ num_bits, group_size, has_act_order, is_k_full,
1422
+ max_shared_mem)) {
1423
+ return exec_config_t{max_m_blocks, th_config};
1424
+ }
1425
}
1426
}
1427
@@ -1574,10 +1594,12 @@ void marlin_mm_f16i4(const void *A, const void *B, void *C, void *s,
1594
}
1595
CALL_IF(4, 32, 2, 256)
1596
CALL_IF(4, 16, 4, 256)
1597
+ CALL_IF(4, 8, 8, 256)
1598
CALL_IF(4, 8, 4, 128)
1599
CALL_IF(4, 4, 8, 128)
1600
CALL_IF(8, 32, 2, 256)
1601
CALL_IF(8, 16, 4, 256)
1602
+ CALL_IF(8, 8, 8, 256)
1603
CALL_IF(8, 8, 4, 128)
1604
CALL_IF(8, 4, 8, 128)
1605
else {