A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[BugFix][Kernel]: fix illegal memory access in causal_conv1d when conv_states is None (#10928)
Signed-off-by: xffxff <1247714429@qq.com>
zhou fan committed
Dec 8, 2024 at 01:21 UTC
78029b34ed1be46baf06f92c9e971ea1961d0867
2 files changed
+23
-18
csrc/mamba/causal_conv1d/causal_conv1d.cu
+1
-1
@@ -424,7 +424,7 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
424
// and the one before it (chunk = n_chunks - 1 and chunk = n_chunks - 2),
425
// (which occurs when `final_state_position` is a non-positivie index)
426
// we load the correct data from smem_exchange from both chunks, the last chunk iteration and the one before it
427
- if (final_state_position < 0 && seqlen > kWidth){
427
+ if (conv_states != nullptr && final_state_position < 0 && seqlen > kWidth){
428
input_t vals_load[kNElts] = {0};
429
if ((chunk == n_chunks - 2) && (tidx == kNThreads - 1)){
430
// chunk = n_chunks - 2, a segment of the final state sits in the last index
tests/kernels/test_causal_conv1d.py
+22
-17
@@ -149,13 +149,14 @@ def causal_conv1d_opcheck_fn(x: torch.Tensor,
149
@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
150
@pytest.mark.parametrize("silu_activation", [True])
151
@pytest.mark.parametrize("has_bias", [True])
152
+@pytest.mark.parametrize("has_initial_state", [True, False])
153
@pytest.mark.parametrize("width", [4])
154
@pytest.mark.parametrize(
155
'seqlen', [1, 8, 16, 32, 64, 128, 256, 512, 784, 1024, 1025, 2048, 4096])
156
@pytest.mark.parametrize('dim', [64])
157
@pytest.mark.parametrize('batch', [1])
158
def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
158
- itype):
159
+ has_initial_state, itype):
160
device = "cuda"
161
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
162
if itype == torch.bfloat16:
@@ -167,11 +168,18 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
168
169
weight = torch.randn(dim, width, device=device, dtype=itype)
170
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
170
- initial_states = torch.randn(batch,
171
- dim,
172
- width - 1,
173
- device=device,
174
- dtype=itype)
171
+ if has_initial_state:
172
+ initial_states = torch.randn(batch,
173
+ dim,
174
+ width - 1,
175
+ device=device,
176
+ dtype=itype)
177
+ has_initial_state_tensor = torch.ones(batch,
178
+ dtype=torch.bool,
179
+ device=x.device)
180
+ else:
181
+ initial_states = None
182
+ has_initial_state_tensor = None
183
x_ref = x.clone()
184
weight_ref = weight.clone()
185
bias_ref = bias.clone() if bias is not None else None
@@ -183,9 +191,7 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
191
bias,
192
activation=activation,
193
conv_states=initial_states,
186
- has_initial_state=torch.ones(batch,
187
- dtype=torch.bool,
188
- device=x.device))
194
+ has_initial_state=has_initial_state_tensor)
195
out_ref, final_states_ref = causal_conv1d_ref(
196
x_ref,
197
weight_ref,
@@ -193,11 +199,12 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
199
initial_states=initial_states_ref,
200
return_final_states=True,
201
activation=activation)
196
- assert initial_states is not None and final_states_ref is not None
197
- assert torch.allclose(initial_states,
198
- final_states_ref,
199
- rtol=rtol,
200
- atol=atol)
202
+ if has_initial_state:
203
+ assert initial_states is not None and final_states_ref is not None
204
+ assert torch.allclose(initial_states,
205
+ final_states_ref,
206
+ rtol=rtol,
207
+ atol=atol)
208
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
209
210
causal_conv1d_opcheck_fn(x,
@@ -205,9 +212,7 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
212
bias,
213
activation=activation,
214
conv_states=initial_states,
208
- has_initial_state=torch.ones(batch,
209
- dtype=torch.bool,
210
- device=x.device))
215
+ has_initial_state=has_initial_state_tensor)
216
217
218
@pytest.mark.parametrize("itype", [torch.bfloat16])