A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Kernel] Explicitly specify other value in tl.load calls (#9014)
Signed-off-by: Angus Wang <wangjadehao@gmail.com>
Angus Wang committed
Nov 18, 2024 at 11:39 UTC
c2170a5b395acb9f5f4ce8425c3be18aacb67513
6 files changed
+35
-14
vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py
+10
-3
@@ -157,19 +157,22 @@ def _fwd_kernel_inner(
157
k = tl.load(
158
k_ptrs + start_n * stride_kt,
159
mask=offs_n[None, :] + start_n < k_seqlen,
160
+ other=0.0,
161
)
162
else:
163
k = tl.load(
164
k_ptrs + start_n * stride_kt,
165
mask=(offs_n[None, :] + start_n < k_seqlen) &
166
(offs_d[:, None] < D_HEAD),
167
+ other=0.0,
168
)
169
else:
170
if EVEN_D:
171
k = tl.load(k_ptrs + start_n * stride_kt)
172
else:
173
k = tl.load(k_ptrs + start_n * stride_kt,
172
- mask=offs_d[:, None] < D_HEAD)
174
+ mask=offs_d[:, None] < D_HEAD,
175
+ other=0.0)
176
177
qk = tl.zeros([BLOCK_M_LOADING, BLOCK_N], dtype=tl.float32)
178
qk += tl.dot(q, k)
@@ -200,19 +203,22 @@ def _fwd_kernel_inner(
203
v = tl.load(
204
v_ptrs + start_n * stride_vt,
205
mask=offs_n[:, None] + start_n < k_seqlen,
206
+ other=0.0,
207
)
208
else:
209
v = tl.load(
210
v_ptrs + start_n * stride_vt,
211
mask=(offs_n[:, None] + start_n < k_seqlen) &
212
(offs_d[None, :] < D_HEAD),
213
+ other=0.0,
214
)
215
else:
216
if EVEN_D:
217
v = tl.load(v_ptrs + start_n * stride_vt)
218
else:
219
v = tl.load(v_ptrs + start_n * stride_vt,
215
- mask=offs_d[None, :] < D_HEAD)
220
+ mask=offs_d[None, :] < D_HEAD,
221
+ other=0.0)
222
223
acc += tl.dot(p, v)
224
@@ -318,12 +324,13 @@ def _fwd_kernel_batch_inference(
324
q = tl.load(
325
Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
326
mask=offs_m[:, None] < q_seqlen,
327
+ other=0.0,
328
)
329
else:
330
q = tl.load(
331
Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
332
mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD),
326
- other=0,
333
+ other=0.0,
334
)
335
336
sparse_crow_ptr = (layout_crow_ptr + off_h * layout_crow_stride_h +
vllm/lora/ops/bgmv_expand.py
+3
-1
@@ -75,7 +75,9 @@ def _bgmv_expand_kernel(
75
other=0.0,
76
) # [BLOCK_N,BLOCK_K]
77
if ADD_INPUTS:
78
- tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
78
+ tiled_out = tl.load(c_ptr + current_n * cn_stride,
79
+ mask=c_mask,
80
+ other=0.0)
81
accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
82
else:
83
accumulator = tl.sum(tiled_a * tiled_b, 1)
vllm/lora/ops/bgmv_expand_slice.py
+7
-1
@@ -78,7 +78,13 @@ def _bgmv_expand_slice_kernel(
78
) # [BLOCK_N,BLOCK_K]
79
80
if ADD_INPUTS:
81
- tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
81
+ # explicitly pass in other=None to tell triton that masked values
82
+ # can be uninitialized. This is OK because the later tl.store
83
+ # operation uses the same mask, eliminating the risk of garbage
84
+ # values propagating
85
+ tiled_out = tl.load(c_ptr + current_n * cn_stride,
86
+ mask=c_mask,
87
+ other=None)
88
accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
89
else:
90
accumulator = tl.sum(tiled_a * tiled_b, 1)
vllm/lora/ops/sgmv_expand.py
+4
-1
@@ -88,7 +88,10 @@ def _sgmv_expand_kernel(
88
c_mask = (offset_cm[:, None] <
89
(cur_seq_start + M)) & (offset_cn[None, :] < N)
90
if ADD_INPUTS:
91
- tiled_out = tl.load(c_ptr, mask=c_mask)
91
+ # explicitly pass in other=None to tell triton that masked values
92
+ # can be uninitialized. This is OK because the later tl.store operation
93
+ # uses the same mask, eliminating the risk of garbage values propagating
94
+ tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
95
tiled_c += tiled_out
96
tl.store(c_ptr, tiled_c, mask=c_mask)
97
vllm/lora/ops/sgmv_expand_slice.py
+4
-1
@@ -94,7 +94,10 @@ def _sgmv_expand_slice_kernel(
94
c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :] <
95
(slice_offset + N))
96
if ADD_INPUTS:
97
- tiled_out = tl.load(c_ptr, mask=c_mask)
97
+ # explicitly pass in other=None to tell triton that masked values
98
+ # can be uninitialized. This is OK because the later tl.store operation
99
+ # uses the same mask, eliminating the risk of garbage values propagating
100
+ tiled_out = tl.load(c_ptr, mask=c_mask, other=None)
101
tiled_c += tiled_out
102
tl.store(c_ptr, tiled_c, mask=c_mask)
103
vllm/model_executor/layers/quantization/awq_triton.py
+7
-7
@@ -42,7 +42,7 @@ def awq_dequantize_kernel(
42
result_masks = result_masks_y[:, None] & result_masks_x[None, :]
43
44
# Load the weights.
45
- iweights = tl.load(qweight_ptr + offsets, masks)
45
+ iweights = tl.load(qweight_ptr + offsets, masks, 0.0)
46
iweights = tl.interleave(iweights, iweights)
47
iweights = tl.interleave(iweights, iweights)
48
iweights = tl.interleave(iweights, iweights)
@@ -71,7 +71,7 @@ def awq_dequantize_kernel(
71
zero_masks = zero_masks_y[:, None] & zero_masks_x[None, :]
72
73
# Load the zeros.
74
- zeros = tl.load(zeros_ptr + zero_offsets, zero_masks)
74
+ zeros = tl.load(zeros_ptr + zero_offsets, zero_masks, 0.0)
75
zeros = tl.interleave(zeros, zeros)
76
zeros = tl.interleave(zeros, zeros)
77
zeros = tl.interleave(zeros, zeros)
@@ -91,7 +91,7 @@ def awq_dequantize_kernel(
91
scale_masks = scale_masks_y[:, None] & scale_masks_x[None, :]
92
93
# Load the scales.
94
- scales = tl.load(scales_ptr + scale_offsets, scale_masks)
94
+ scales = tl.load(scales_ptr + scale_offsets, scale_masks, 0.0)
95
scales = tl.broadcast_to(scales, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8))
96
97
# Dequantize.
@@ -165,10 +165,10 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
165
for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K)):
166
masks_k = offsets_k < K
167
masks_a = masks_am[:, None] & masks_k[None, :]
168
- a = tl.load(a_ptrs, mask=masks_a)
168
+ a = tl.load(a_ptrs, mask=masks_a, other=0.0)
169
170
masks_b = masks_k[:, None] & masks_bn[None, :]
171
- b = tl.load(b_ptrs, mask=masks_b)
171
+ b = tl.load(b_ptrs, mask=masks_b, other=0.0)
172
b = tl.interleave(b, b)
173
b = tl.interleave(b, b)
174
b = tl.interleave(b, b)
@@ -181,7 +181,7 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
181
masks_zk = offsets_szk < K // group_size
182
masks_z = masks_zk[:, None] & masks_zn[None, :]
183
zeros_ptrs = zeros_ptr + offsets_z
184
- zeros = tl.load(zeros_ptrs, mask=masks_z)
184
+ zeros = tl.load(zeros_ptrs, mask=masks_z, other=0.0)
185
zeros = tl.interleave(zeros, zeros)
186
zeros = tl.interleave(zeros, zeros)
187
zeros = tl.interleave(zeros, zeros)
@@ -191,7 +191,7 @@ def awq_gemm_kernel(a_ptr, b_ptr, c_ptr, zeros_ptr, scales_ptr, M, N, K,
191
masks_sk = offsets_szk < K // group_size
192
masks_s = masks_sk[:, None] & masks_sn[None, :]
193
scales_ptrs = scales_ptr + offsets_s
194
- scales = tl.load(scales_ptrs, mask=masks_s)
194
+ scales = tl.load(scales_ptrs, mask=masks_s, other=0.0)
195
scales = tl.broadcast_to(scales, (BLOCK_SIZE_K, BLOCK_SIZE_N))
196
197
b = (b >> shifts) & 0xF