@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c2170a5b39
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Kernel] Explicitly specify other value in tl.load calls (#9014)

Signed-off-by: Angus Wang <wangjadehao@gmail.com>

Angus Wang committed Nov 18, 2024 at 11:39 UTC c2170a5b395acb9f5f4ce8425c3be18aacb67513
6 files changed +35 -14
vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py
+10 -3
@@ -157,19 +157,22 @@ def _fwd_kernel_inner(
157 k = tl.load(
158 k_ptrs + start_n * stride_kt,
159 mask=offs_n[None, :] + start_n < k_seqlen,
160 + other=0.0,
161 )
162 else:
163 k = tl.load(
164 k_ptrs + start_n * stride_kt,
165 mask=(offs_n[None, :] + start_n < k_seqlen) &
166 (offs_d[:, None] < D_HEAD),
167 + other=0.0,
168 )
169 else:
170 if EVEN_D:
171 k = tl.load(k_ptrs + start_n * stride_kt)
172 else:
173 k = tl.load(k_ptrs + start_n * stride_kt,
172 - mask=offs_d[:, None] < D_HEAD)
174 + mask=offs_d[:, None] < D_HEAD,
175 + other=0.0)
176
177 qk = tl.zeros([BLOCK_M_LOADING, BLOCK_N], dtype=tl.float32)
178 qk += tl.dot(q, k)
@@ -200,19 +203,22 @@ def _fwd_kernel_inner(
203 v = tl.load(
204 v_ptrs + start_n * stride_vt,
205 mask=offs_n[:, None] + start_n < k_seqlen,
206 + other=0.0,
207 )
208 else:
209 v = tl.load(
210 v_ptrs + start_n * stride_vt,
211 mask=(offs_n[:, None] + start_n < k_seqlen) &
212 (offs_d[None, :] < D_HEAD),
213 + other=0.0,
214 )
215 else:
216 if EVEN_D:
217 v = tl.load(v_ptrs + start_n * stride_vt)
218 else:
219 v = tl.load(v_ptrs + start_n * stride_vt,
215 - mask=offs_d[None, :] < D_HEAD)
220 + mask=offs_d[None, :] < D_HEAD,
221 + other=0.0)
222
223 acc += tl.dot(p, v)
224
@@ -318,12 +324,13 @@ def _fwd_kernel_batch_inference(
324 q = tl.load(
325 Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
326 mask=offs_m[:, None] < q_seqlen,
327 + other=0.0,
328 )
329 else:
330 q = tl.load(
331 Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
332 mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD),
326 - other=0,
333 + other=0.0,
334 )
335
336 sparse_crow_ptr = (layout_crow_ptr + off_h * layout_crow_stride_h +
vllm/lora/ops/bgmv_expand.py
+3 -1
@@ -75,7 +75,9 @@ def _bgmv_expand_kernel(
75 other=0.0,
76 ) # [BLOCK_N,BLOCK_K]
77 if ADD_INPUTS:
78 - tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
78 + tiled_out = tl.load(c_ptr + current_n * cn_stride,
79 + mask=c_mask,
80 + other=0.0)
81 accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
82 else:
83 accumulator = tl.sum(tiled_a * tiled_b, 1)
vllm/lora/ops/bgmv_expand_slice.py
+7 -1
@@ -78,7 +78,13 @@ def _bgmv_expand_slice_kernel(
78 ) # [BLOCK_N,BLOCK_K]
79
80 if ADD_INPUTS:
81 - tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
81 + # explicitly pass in other=None to tell triton that masked values
82 + # can be uninitialized. This is OK because the later tl.store
83 + # operation uses the same mask, eliminating the risk of garbage
84 + # values propagating
85 + tiled_out = tl.load(c_ptr + current_n * cn_stride,
86 + mask=c_mask,
87 + other=None)
88 accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
89 else:
90 accumulator = tl.sum(tiled_a * tiled_b, 1)
vllm/lora/ops/sgmv_expand.py
+4 -1
@@ -88,7 +88,10 @@ def _sgmv_expand_kernel(
88 c_mask = (offset_cm[:, None] <
89 (cur_seq_start + M)) & (offset_cn[None, :] < N)
90 if ADD_INPUTS:
91 - tiled_out = tl.load(c_ptr, mask=c_mask)
91 + # explicitly pass in other=None to tell triton that masked values
92 + # can be uninitialized. This is OK because the later tl.store operation
93 + # uses the same mask, eliminating the risk of garbage values propagating
94 + tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
95 tiled_c += tiled_out
96 tl.store(c_ptr, tiled_c, mask=c_mask)
97
vllm/lora/ops/sgmv_expand_slice.py
+4 -1
@@ -94,7 +94,10 @@ def _sgmv_expand_slice_kernel(
94 c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :] <
95 (slice_offset + N))
96 if ADD_INPUTS:
97 - tiled_out = tl.load(c_ptr, mask=c_mask)
97 + # explicitly pass in other=None to tell triton that masked values
98 + # can be uninitialized. This is OK because the later tl.store operation
99 + # uses the same mask, eliminating the risk of garbage values propagating
100 + tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
101 tiled_c += tiled_out
102 tl.store(c_ptr, tiled_c, mask=c_mask)
103
vllm/model_executor/layers/quantization/awq_triton.py
+7 -7
@@ -42,7 +42,7 @@ def awq_dequantize_kernel(
42 result_masks = result_masks_y[:, None] & result_masks_x[None, :]
43
44 # Load the weights.
45 - iweights = tl.load(qweight_ptr + offsets, masks)
45 + iweights = tl.load(qweight_ptr + offsets, masks, 0.0)
46 iweights = tl.interleave(iweights, iweights)
47 iweights = tl.interleave(iweights, iweights)
48 iweights = tl.interleave(iweights, iweights)
@@ -71,7 +71,7 @@ def awq_dequantize_kernel(
71 zero_masks = zero_masks_y[:, None] & zero_masks_x[None, :]
72
73 # Load the zeros.
74 - zeros = tl.load(zeros_ptr + zero_offsets, zero_masks)
74 + zeros = tl.load(zeros_ptr + zero_offsets, zero_masks, 0.0)
75 zeros = tl.interleave(zeros, zeros)
76 zeros = tl.interleave(zeros, zeros)
77 zeros = tl.interleave(zeros, zeros)
@@ -91,7 +91,7 @@ def awq_dequantize_kernel(
91 scale_masks = scale_masks_y[:, None] & scale_masks_x[None, :]
92
93 # Load the scales.
94 - scales = tl.load(scales_ptr + scale_offsets, scale_masks)
94 + scales = tl.load(scales_ptr + scale_offsets, scale_masks, 0.0)
95 scales = tl.broadcast_to(scales, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8))
96
97 # Dequantize.
@@ -165,10 +165,10 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
165 for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K)):
166 masks_k = offsets_k < K
167 masks_a = masks_am[:, None] & masks_k[None, :]
168 - a = tl.load(a_ptrs, mask=masks_a)
168 + a = tl.load(a_ptrs, mask=masks_a, other=0.0)
169
170 masks_b = masks_k[:, None] & masks_bn[None, :]
171 - b = tl.load(b_ptrs, mask=masks_b)
171 + b = tl.load(b_ptrs, mask=masks_b, other=0.0)
172 b = tl.interleave(b, b)
173 b = tl.interleave(b, b)
174 b = tl.interleave(b, b)
@@ -181,7 +181,7 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
181 masks_zk = offsets_szk < K // group_size
182 masks_z = masks_zk[:, None] & masks_zn[None, :]
183 zeros_ptrs = zeros_ptr + offsets_z
184 - zeros = tl.load(zeros_ptrs, mask=masks_z)
184 + zeros = tl.load(zeros_ptrs, mask=masks_z, other=0.0)
185 zeros = tl.interleave(zeros, zeros)
186 zeros = tl.interleave(zeros, zeros)
187 zeros = tl.interleave(zeros, zeros)
@@ -191,7 +191,7 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
191 masks_sk = offsets_szk < K // group_size
192 masks_s = masks_sk[:, None] & masks_sn[None, :]
193 scales_ptrs = scales_ptr + offsets_s
194 - scales = tl.load(scales_ptrs, mask=masks_s)
194 + scales = tl.load(scales_ptrs, mask=masks_s, other=0.0)
195 scales = tl.broadcast_to(scales, (BLOCK_SIZE_K, BLOCK_SIZE_N))
196
197 b = (b >> shifts) & 0xF