@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5dba257506
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

Resolve race conditions in Marlin kernel (#11493)

Signed-off-by: wchen61 <wchen61@foxmail.com>

wchen61 committed Jan 3, 2025 at 06:58 UTC 5dba2575065f5e27d468f2776e3d460a21d916e6
1 file changed +21 -19
csrc/quantization/gptq_marlin/gptq_marlin.cu
+21 -19
@@ -834,6 +834,7 @@ __global__ void Marlin(
834 int4* sh_g_idx = sh_b + (stages * b_sh_stage);
835 int4* sh_zp = sh_g_idx + (stages * g_idx_stage);
836 int4* sh_s = sh_zp + (stages * zp_sh_stage);
837 + int4* sh_red = sh_s + (stages * s_sh_stage);
838
839 // Register storage for double buffer of shared memory reads.
840 FragA frag_a[2][thread_m_blocks];
@@ -932,11 +933,11 @@ __global__ void Marlin(
933 int4* sh_s_stage = sh_s + s_sh_stage * pipe;
934
935 if constexpr (group_blocks >= thread_k_blocks) {
936 + if (s_sh_wr_pred) {
937 + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
938 + }
939 // Only fetch scales if this tile starts a new group
936 - if (pipe % (group_blocks / thread_k_blocks) == 0) {
937 - if (s_sh_wr_pred) {
938 - cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
939 - }
940 + if ((pipe + 1) % (group_blocks / thread_k_blocks) == 0) {
941 s_gl_rd += s_gl_rd_delta;
942 }
943 } else {
@@ -1038,9 +1039,7 @@ __global__ void Marlin(
1039 // No act-order case
1040 if constexpr (group_blocks != -1) {
1041 if constexpr (group_blocks >= thread_k_blocks) {
1041 - int4* sh_s_stage =
1042 - sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
1043 - (pipe / (group_blocks / thread_k_blocks)));
1042 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
1043 reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
1044 } else {
1045 int warp_id = threadIdx.x / 32;
@@ -1339,15 +1338,15 @@ __global__ void Marlin(
1338 int red_sh_wr =
1339 red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
1340 if (i < red_off) {
1342 - float* c_rd =
1343 - reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
1344 - float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
1341 + float* c_rd = reinterpret_cast<float*>(
1342 + &sh_red[red_sh_delta * j + red_sh_rd]);
1343 + float* c_wr = reinterpret_cast<float*>(&sh_red[red_sh_wr]);
1344 #pragma unroll
1345 for (int k = 0; k < 4; k++)
1346 reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
1347 c_rd[k] + c_wr[k];
1348 }
1350 - sh[red_sh_wr] =
1349 + sh_red[red_sh_wr] =
1350 reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
1351 }
1352 }
@@ -1357,7 +1356,7 @@ __global__ void Marlin(
1356 #pragma unroll
1357 for (int i = 0; i < 4 * 2; i++) {
1358 float* c_rd =
1360 - reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
1359 + reinterpret_cast<float*>(&sh_red[red_sh_delta * i + red_sh_rd]);
1360 #pragma unroll
1361 for (int j = 0; j < 4; j++)
1362 reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
@@ -1397,7 +1396,7 @@ __global__ void Marlin(
1396 #pragma unroll
1397 for (int i = 0; i < thread_m_blocks * 4; i++) {
1398 cp_async4_pred(
1400 - &sh[c_sh_wr + c_sh_wr_delta * i],
1399 + &sh_red[c_sh_wr + c_sh_wr_delta * i],
1400 &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
1401 c_gl_wr_delta_i * (i % 2)],
1402 i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
@@ -1410,7 +1409,7 @@ __global__ void Marlin(
1409 for (int i = 0; i < thread_m_blocks * 4; i++) {
1410 if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
1411 if (!first) {
1413 - int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
1412 + int4 c_red = sh_red[c_sh_wr + i * c_sh_wr_delta];
1413 #pragma unroll
1414 for (int j = 0; j < 2 * 4; j++) {
1415 reinterpret_cast<float*>(
@@ -1461,10 +1460,10 @@ __global__ void Marlin(
1460 float* frag_c_ptr = reinterpret_cast<float*>(&frag_c);
1461 #pragma unroll
1462 for (int k = 0; k < th_size; k++) {
1464 - sh[threadIdx.x] =
1463 + sh_red[threadIdx.x] =
1464 C_tmp[c_cur_offset + active_threads * k + threadIdx.x];
1465
1467 - float* sh_c_ptr = reinterpret_cast<float*>(&sh[threadIdx.x]);
1466 + float* sh_c_ptr = reinterpret_cast<float*>(&sh_red[threadIdx.x]);
1467 #pragma unroll
1468 for (int f = 0; f < 4; f++) {
1469 frag_c_ptr[k * 4 + f] += sh_c_ptr[f];
@@ -1515,7 +1514,7 @@ __global__ void Marlin(
1514 res = __hmul2(res, s[0]);
1515 }
1516
1518 - ((scalar_t2*)sh)[idx] = res;
1517 + ((scalar_t2*)sh_red)[idx] = res;
1518 };
1519
1520 if (threadIdx.x / 32 < thread_n_blocks / 4) {
@@ -1543,7 +1542,7 @@ __global__ void Marlin(
1542 i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1543 i++) {
1544 if (c_gl_wr < c_gl_wr_end) {
1546 - C[c_gl_wr] = sh[c_sh_rd];
1545 + C[c_gl_wr] = sh_red[c_sh_rd];
1546 c_gl_wr += c_gl_wr_delta;
1547 c_sh_rd += c_sh_rd_delta;
1548 }
@@ -1865,9 +1864,12 @@ bool is_valid_cache_size(thread_config_t const& th_config, int max_m_blocks,
1864
1865 float pipe_size = (a_size + b_size) * pipe_stages;
1866
1867 + float reduce_size = max(th_config.num_threads * 32 * 4,
1868 + (tb_n / 64) * 32 * (tb_max_m / 16) * 4 * 2 * 4 * 2);
1869 +
1870 TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1871
1870 - return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1872 + return pipe_size + reduce_size < 0.95f * (max_shared_mem - scales_cache_size);
1873 }
1874
1875 bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,