A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Fix marlin divide-by-zero warnings (#6904)
Tyler Michael Smith committed
Jul 29, 2024 at 21:26 UTC
61a97c32f64641738d2cc623708f28046768224e
3 files changed
+58
-39
csrc/quantization/gptq_marlin/gptq_marlin.cu
+32
-29
@@ -1128,44 +1128,47 @@ __global__ void Marlin(
1128
};
1129
1130
auto fetch_zp_to_registers = [&](int k, int full_pipe) {
1131
- if constexpr (!has_zp) {
1132
- return;
1133
- }
1131
+ if constexpr (has_zp) {
1132
+ // This code does not handle group_blocks == 0,
1133
+ // which signifies act_order.
1134
+ // has_zp implies AWQ, which doesn't have act_order,
1135
+ static_assert(group_blocks != 0);
1136
1135
- int pipe = full_pipe % stages;
1137
+ int pipe = full_pipe % stages;
1138
1137
- if constexpr (group_blocks == -1) {
1138
- for (int i = 0; i < num_ints_per_thread; i++) {
1139
- frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
1140
- }
1139
+ if constexpr (group_blocks == -1) {
1140
+ for (int i = 0; i < num_ints_per_thread; i++) {
1141
+ frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
1142
+ }
1143
1142
- } else if constexpr (group_blocks >= thread_k_blocks) {
1143
- int4* sh_zp_stage =
1144
- sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1145
- (pipe / (group_blocks / thread_k_blocks)));
1146
- for (int i = 0; i < num_ints_per_thread; i++) {
1147
- frag_qzp[k % 2][i] =
1148
- (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1149
- }
1150
- } else {
1151
- int warp_id = threadIdx.x / 32;
1152
- int n_warps = thread_n_blocks / 4;
1144
+ } else if constexpr (group_blocks >= thread_k_blocks) {
1145
+ int4* sh_zp_stage =
1146
+ sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1147
+ (pipe / (group_blocks / thread_k_blocks)));
1148
+ for (int i = 0; i < num_ints_per_thread; i++) {
1149
+ frag_qzp[k % 2][i] =
1150
+ (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1151
+ }
1152
+ } else {
1153
+ int warp_id = threadIdx.x / 32;
1154
+ int n_warps = thread_n_blocks / 4;
1155
1154
- int warp_row = warp_id / n_warps;
1156
+ int warp_row = warp_id / n_warps;
1157
1156
- int cur_k = warp_row * 16;
1157
- cur_k += k_iter_size * (k % b_sh_wr_iters);
1158
+ int cur_k = warp_row * 16;
1159
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
1160
1159
- int k_blocks = cur_k / 16;
1160
- int cur_group_id = k_blocks / group_blocks;
1161
+ int k_blocks = cur_k / 16;
1162
+ int cur_group_id = k_blocks / group_blocks;
1163
1162
- int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1164
+ int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1165
1164
- sh_zp_stage += cur_group_id * zp_sh_stride;
1166
+ sh_zp_stage += cur_group_id * zp_sh_stride;
1167
1166
- for (int i = 0; i < num_ints_per_thread; i++) {
1167
- frag_qzp[k % 2][i] =
1168
- (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1168
+ for (int i = 0; i < num_ints_per_thread; i++) {
1169
+ frag_qzp[k % 2][i] =
1170
+ (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1171
+ }
1172
}
1173
}
1174
};
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+13
-5
@@ -452,10 +452,15 @@ __global__ void Marlin(
452
B_ptr[i] += b_gl_rd_delta_o;
453
}
454
// Only fetch scales if this tile starts a new group
455
- if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
456
- int4* sh_s_stage = sh_s + s_sh_stage * pipe;
457
- if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
458
- s_gl_rd += s_gl_rd_delta;
455
+ if constexpr (group_blocks != -1) {
456
+ // This assumes group_blocks >= thread_k_blocks
457
+ // and would need to be modified to support smaller groups.
458
+ static_assert(group_blocks >= thread_k_blocks);
459
+ if (pipe % (group_blocks / thread_k_blocks) == 0) {
460
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
461
+ if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
462
+ s_gl_rd += s_gl_rd_delta;
463
+ }
464
}
465
}
466
// Insert a fence even when we are winding down the pipeline to ensure that
@@ -480,7 +485,10 @@ __global__ void Marlin(
485
// however, this does not seem to be a significant bottleneck, while some
486
// theoretically better attempts have lead to bad instruction ordering by
487
// the compiler and correspondingly a noticeable drop in performance.
483
- if (group_blocks != -1) {
488
+ if constexpr (group_blocks != -1) {
489
+ // This assumes group_blocks >= thread_k_blocks
490
+ // and would need to be modified to support smaller groups.
491
+ static_assert(group_blocks >= thread_k_blocks);
492
int4* sh_s_stage =
493
sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
494
(pipe / (group_blocks / thread_k_blocks)));
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+13
-5
@@ -404,10 +404,15 @@ __global__ void Marlin_24(
404
meta_ptr[i] += m_gl_rd_delta_o;
405
}
406
// Only fetch scales if this tile starts a new group
407
- if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
408
- int4* sh_s_stage = sh_s + s_sh_stage * pipe;
409
- if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
410
- s_gl_rd += s_gl_rd_delta;
407
+ if constexpr (group_blocks != -1) {
408
+ // This assumes group_blocks >= thread_k_blocks
409
+ // and would need to be modified to support smaller groups.
410
+ static_assert(group_blocks >= thread_k_blocks);
411
+ if (pipe % (group_blocks / thread_k_blocks) == 0) {
412
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
413
+ if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
414
+ s_gl_rd += s_gl_rd_delta;
415
+ }
416
}
417
}
418
// Insert a fence even when we are winding down the pipeline to ensure that
@@ -432,7 +437,10 @@ __global__ void Marlin_24(
437
// however, this does not seem to be a significant bottleneck, while some
438
// theoretically better attempts have lead to bad instruction ordering by
439
// the compiler and correspondingly a noticeable drop in performance.
435
- if (group_blocks != -1) {
440
+ if constexpr (group_blocks != -1) {
441
+ // This assumes group_blocks >= thread_k_blocks
442
+ // and would need to be modified to support smaller groups.
443
+ static_assert(group_blocks >= thread_k_blocks);
444
int4* sh_s_stage =
445
sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
446
(pipe / (group_blocks / thread_k_blocks)));