@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 78029b34ed
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[BugFix][Kernel]: fix illegal memory access in causal_conv1d when conv_states is None (#10928)

Signed-off-by: xffxff <1247714429@qq.com>

zhou fan committed Dec 8, 2024 at 01:21 UTC 78029b34ed1be46baf06f92c9e971ea1961d0867
2 files changed +23 -18
csrc/mamba/causal_conv1d/causal_conv1d.cu
+1 -1
@@ -424,7 +424,7 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
424 // and the one before it (chunk = n_chunks - 1 and chunk = n_chunks - 2),
425 // (which occurs when `final_state_position` is a non-positivie index)
426 // we load the correct data from smem_exchange from both chunks, the last chunk iteration and the one before it
427 - if (final_state_position < 0 && seqlen > kWidth){
427 + if (conv_states != nullptr && final_state_position < 0 && seqlen > kWidth){
428 input_t vals_load[kNElts] = {0};
429 if ((chunk == n_chunks - 2) && (tidx == kNThreads - 1)){
430 // chunk = n_chunks - 2, a segment of the final state sits in the last index
tests/kernels/test_causal_conv1d.py
+22 -17
@@ -149,13 +149,14 @@ def causal_conv1d_opcheck_fn(x: torch.Tensor,
149 @pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
150 @pytest.mark.parametrize("silu_activation", [True])
151 @pytest.mark.parametrize("has_bias", [True])
152 +@pytest.mark.parametrize("has_initial_state", [True, False])
153 @pytest.mark.parametrize("width", [4])
154 @pytest.mark.parametrize(
155 'seqlen', [1, 8, 16, 32, 64, 128, 256, 512, 784, 1024, 1025, 2048, 4096])
156 @pytest.mark.parametrize('dim', [64])
157 @pytest.mark.parametrize('batch', [1])
158 def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
158 - itype):
159 + has_initial_state, itype):
160 device = "cuda"
161 rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
162 if itype == torch.bfloat16:
@@ -167,11 +168,18 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
168
169 weight = torch.randn(dim, width, device=device, dtype=itype)
170 bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
170 - initial_states = torch.randn(batch,
171 - dim,
172 - width - 1,
173 - device=device,
174 - dtype=itype)
171 + if has_initial_state:
172 + initial_states = torch.randn(batch,
173 + dim,
174 + width - 1,
175 + device=device,
176 + dtype=itype)
177 + has_initial_state_tensor = torch.ones(batch,
178 + dtype=torch.bool,
179 + device=x.device)
180 + else:
181 + initial_states = None
182 + has_initial_state_tensor = None
183 x_ref = x.clone()
184 weight_ref = weight.clone()
185 bias_ref = bias.clone() if bias is not None else None
@@ -183,9 +191,7 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
191 bias,
192 activation=activation,
193 conv_states=initial_states,
186 - has_initial_state=torch.ones(batch,
187 - dtype=torch.bool,
188 - device=x.device))
194 + has_initial_state=has_initial_state_tensor)
195 out_ref, final_states_ref = causal_conv1d_ref(
196 x_ref,
197 weight_ref,
@@ -193,11 +199,12 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
199 initial_states=initial_states_ref,
200 return_final_states=True,
201 activation=activation)
196 - assert initial_states is not None and final_states_ref is not None
197 - assert torch.allclose(initial_states,
198 - final_states_ref,
199 - rtol=rtol,
200 - atol=atol)
202 + if has_initial_state:
203 + assert initial_states is not None and final_states_ref is not None
204 + assert torch.allclose(initial_states,
205 + final_states_ref,
206 + rtol=rtol,
207 + atol=atol)
208 assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
209
210 causal_conv1d_opcheck_fn(x,
@@ -205,9 +212,7 @@ def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
212 bias,
213 activation=activation,
214 conv_states=initial_states,
208 - has_initial_state=torch.ones(batch,
209 - dtype=torch.bool,
210 - device=x.device))
215 + has_initial_state=has_initial_state_tensor)
216
217
218 @pytest.mark.parametrize("itype", [torch.bfloat16])