@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 61a97c32f6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Fix marlin divide-by-zero warnings (#6904)

Tyler Michael Smith committed Jul 29, 2024 at 21:26 UTC 61a97c32f64641738d2cc623708f28046768224e
3 files changed +58 -39
csrc/quantization/gptq_marlin/gptq_marlin.cu
+32 -29
@@ -1128,44 +1128,47 @@ __global__ void Marlin(
1128 };
1129
1130 auto fetch_zp_to_registers = [&](int k, int full_pipe) {
1131 - if constexpr (!has_zp) {
1132 - return;
1133 - }
1131 + if constexpr (has_zp) {
1132 + // This code does not handle group_blocks == 0,
1133 + // which signifies act_order.
1134 + // has_zp implies AWQ, which doesn't have act_order,
1135 + static_assert(group_blocks != 0);
1136
1135 - int pipe = full_pipe % stages;
1137 + int pipe = full_pipe % stages;
1138
1137 - if constexpr (group_blocks == -1) {
1138 - for (int i = 0; i < num_ints_per_thread; i++) {
1139 - frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
1140 - }
1139 + if constexpr (group_blocks == -1) {
1140 + for (int i = 0; i < num_ints_per_thread; i++) {
1141 + frag_qzp[k % 2][i] = (reinterpret_cast<int*>(sh_zp))[zp_sh_rd + i];
1142 + }
1143
1142 - } else if constexpr (group_blocks >= thread_k_blocks) {
1143 - int4* sh_zp_stage =
1144 - sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1145 - (pipe / (group_blocks / thread_k_blocks)));
1146 - for (int i = 0; i < num_ints_per_thread; i++) {
1147 - frag_qzp[k % 2][i] =
1148 - (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1149 - }
1150 - } else {
1151 - int warp_id = threadIdx.x / 32;
1152 - int n_warps = thread_n_blocks / 4;
1144 + } else if constexpr (group_blocks >= thread_k_blocks) {
1145 + int4* sh_zp_stage =
1146 + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) *
1147 + (pipe / (group_blocks / thread_k_blocks)));
1148 + for (int i = 0; i < num_ints_per_thread; i++) {
1149 + frag_qzp[k % 2][i] =
1150 + (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1151 + }
1152 + } else {
1153 + int warp_id = threadIdx.x / 32;
1154 + int n_warps = thread_n_blocks / 4;
1155
1154 - int warp_row = warp_id / n_warps;
1156 + int warp_row = warp_id / n_warps;
1157
1156 - int cur_k = warp_row * 16;
1157 - cur_k += k_iter_size * (k % b_sh_wr_iters);
1158 + int cur_k = warp_row * 16;
1159 + cur_k += k_iter_size * (k % b_sh_wr_iters);
1160
1159 - int k_blocks = cur_k / 16;
1160 - int cur_group_id = k_blocks / group_blocks;
1161 + int k_blocks = cur_k / 16;
1162 + int cur_group_id = k_blocks / group_blocks;
1163
1162 - int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1164 + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe;
1165
1164 - sh_zp_stage += cur_group_id * zp_sh_stride;
1166 + sh_zp_stage += cur_group_id * zp_sh_stride;
1167
1166 - for (int i = 0; i < num_ints_per_thread; i++) {
1167 - frag_qzp[k % 2][i] =
1168 - (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1168 + for (int i = 0; i < num_ints_per_thread; i++) {
1169 + frag_qzp[k % 2][i] =
1170 + (reinterpret_cast<int*>(sh_zp_stage))[zp_sh_rd + i];
1171 + }
1172 }
1173 }
1174 };
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+13 -5
@@ -452,10 +452,15 @@ __global__ void Marlin(
452 B_ptr[i] += b_gl_rd_delta_o;
453 }
454 // Only fetch scales if this tile starts a new group
455 - if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
456 - int4* sh_s_stage = sh_s + s_sh_stage * pipe;
457 - if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
458 - s_gl_rd += s_gl_rd_delta;
455 + if constexpr (group_blocks != -1) {
456 + // This assumes group_blocks >= thread_k_blocks
457 + // and would need to be modified to support smaller groups.
458 + static_assert(group_blocks >= thread_k_blocks);
459 + if (pipe % (group_blocks / thread_k_blocks) == 0) {
460 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
461 + if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
462 + s_gl_rd += s_gl_rd_delta;
463 + }
464 }
465 }
466 // Insert a fence even when we are winding down the pipeline to ensure that
@@ -480,7 +485,10 @@ __global__ void Marlin(
485 // however, this does not seem to be a significant bottleneck, while some
486 // theoretically better attempts have lead to bad instruction ordering by
487 // the compiler and correspondingly a noticeable drop in performance.
483 - if (group_blocks != -1) {
488 + if constexpr (group_blocks != -1) {
489 + // This assumes group_blocks >= thread_k_blocks
490 + // and would need to be modified to support smaller groups.
491 + static_assert(group_blocks >= thread_k_blocks);
492 int4* sh_s_stage =
493 sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
494 (pipe / (group_blocks / thread_k_blocks)));
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+13 -5
@@ -404,10 +404,15 @@ __global__ void Marlin_24(
404 meta_ptr[i] += m_gl_rd_delta_o;
405 }
406 // Only fetch scales if this tile starts a new group
407 - if (group_blocks != -1 && pipe % (group_blocks / thread_k_blocks) == 0) {
408 - int4* sh_s_stage = sh_s + s_sh_stage * pipe;
409 - if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
410 - s_gl_rd += s_gl_rd_delta;
407 + if constexpr (group_blocks != -1) {
408 + // This assumes group_blocks >= thread_k_blocks
409 + // and would need to be modified to support smaller groups.
410 + static_assert(group_blocks >= thread_k_blocks);
411 + if (pipe % (group_blocks / thread_k_blocks) == 0) {
412 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
413 + if (s_sh_wr_pred) cp_async4(&sh_s_stage[s_sh_wr], &s[s_gl_rd]);
414 + s_gl_rd += s_gl_rd_delta;
415 + }
416 }
417 }
418 // Insert a fence even when we are winding down the pipeline to ensure that
@@ -432,7 +437,10 @@ __global__ void Marlin_24(
437 // however, this does not seem to be a significant bottleneck, while some
438 // theoretically better attempts have lead to bad instruction ordering by
439 // the compiler and correspondingly a noticeable drop in performance.
435 - if (group_blocks != -1) {
440 + if constexpr (group_blocks != -1) {
441 + // This assumes group_blocks >= thread_k_blocks
442 + // and would need to be modified to support smaller groups.
443 + static_assert(group_blocks >= thread_k_blocks);
444 int4* sh_s_stage =
445 sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
446 (pipe / (group_blocks / thread_k_blocks)));