A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel] Change interface to Mamba causal_conv1d_update for continuous batching (#8012)
Tyler Michael Smith committed
Sep 17, 2024 at 19:44 UTC
8110e44529f431d54b02060528601c0d3e3f7d02
7 files changed
+114
-16
csrc/mamba/causal_conv1d/causal_conv1d.cu
+27
-3
@@ -198,7 +198,8 @@ causal_conv1d_update(const at::Tensor &x,
198
const at::Tensor &conv_state,
199
const at::Tensor &weight,
200
const c10::optional<at::Tensor> &bias_,
201
- bool silu_activation) {
201
+ bool silu_activation,
202
+ const c10::optional<at::Tensor> &conv_state_indices_) {
203
auto input_type = x.scalar_type();
204
auto weight_type = weight.scalar_type();
205
TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -216,7 +217,6 @@ causal_conv1d_update(const at::Tensor &x,
217
const int width = weight.size(-1);
218
219
CHECK_SHAPE(x, batch_size, dim);
219
- CHECK_SHAPE(conv_state, batch_size, dim, width);
220
CHECK_SHAPE(weight, dim, width);
221
222
TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4");
@@ -241,6 +241,22 @@ causal_conv1d_update(const at::Tensor &x,
241
params.conv_state_c_stride = conv_state.stride(1);
242
params.conv_state_l_stride = conv_state.stride(2);
243
244
+ if (conv_state_indices_.has_value()) {
245
+ auto conv_state_indices = conv_state_indices_.value();
246
+ TORCH_CHECK(conv_state_indices.scalar_type() == torch::kInt32)
247
+ TORCH_CHECK(conv_state_indices.is_cuda());
248
+ TORCH_CHECK(conv_state_indices.stride(0) == 1)
249
+ CHECK_SHAPE(conv_state_indices, batch_size);
250
+
251
+ int conv_state_entries = conv_state.size(0);
252
+ CHECK_SHAPE(conv_state, conv_state_entries, dim, width);
253
+
254
+ params.conv_state_indices_ptr = conv_state_indices.data_ptr<int32_t>();
255
+ } else {
256
+ CHECK_SHAPE(conv_state, batch_size, dim, width);
257
+ params.conv_state_indices_ptr = nullptr;
258
+ }
259
+
260
// Otherwise the kernel will be launched from cuda:0 device
261
// Cast to char to avoid compiler warning about narrowing
262
at::cuda::CUDAGuard device_guard{(char)x.get_device()};
@@ -646,8 +662,16 @@ void causal_conv1d_update_kernel(ConvParamsBase params) {
662
const int channel_id = blockIdx.y * kNThreads + tidx;
663
input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
664
+ channel_id * params.x_c_stride;
649
- input_t *conv_state = reinterpret_cast<input_t *>(params.conv_state_ptr) + batch_id * params.conv_state_batch_stride
665
+
666
+ // If params.conv_state_batch_indices is set, then the conv state is gathered from the conv state tensor
667
+ // along the batch axis. Otherwise, the conv state coordinate is the same as the batch id.
668
+ const int conv_state_batch_coord = params.conv_state_indices_ptr == nullptr
669
+ ? batch_id
670
+ : params.conv_state_indices_ptr[batch_id];
671
+ input_t *conv_state = reinterpret_cast<input_t *>(params.conv_state_ptr)
672
+ + conv_state_batch_coord * params.conv_state_batch_stride
673
+ channel_id * params.conv_state_c_stride;
674
+
675
weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr) + channel_id * params.weight_c_stride;
676
input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
677
+ channel_id * params.out_c_stride;
csrc/mamba/causal_conv1d/causal_conv1d.h
+4
@@ -36,6 +36,10 @@ struct ConvParamsBase {
36
37
void *__restrict__ conv_state_ptr;
38
39
+ // For the continuous batching case. Makes it so that the mamba state for
40
+ // the current batch doesn't need to be a contiguous tensor.
41
+ int32_t *__restrict__ conv_state_indices_ptr;
42
+
43
void *__restrict__ seq_idx_ptr;
44
45
// No __restrict__ since initial_states could be the same as final_states.
csrc/ops.h
+4
-5
@@ -222,11 +222,10 @@ std::vector<torch::Tensor> selective_scan_fwd(
222
const c10::optional<torch::Tensor>& index_,
223
const c10::optional<torch::Tensor>& x);
224
225
-at::Tensor causal_conv1d_update(const at::Tensor& x,
226
- const at::Tensor& conv_state,
227
- const at::Tensor& weight,
228
- const c10::optional<at::Tensor>& bias_,
229
- bool silu_activation);
225
+at::Tensor causal_conv1d_update(
226
+ const at::Tensor& x, const at::Tensor& conv_state, const at::Tensor& weight,
227
+ const c10::optional<at::Tensor>& bias, bool silu_activation,
228
+ const c10::optional<at::Tensor>& conv_state_indices);
229
230
at::Tensor causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
231
const c10::optional<at::Tensor>& bias_,
csrc/torch_bindings.cpp
+3
-2
@@ -279,8 +279,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
279
"causal_conv1d_update(Tensor! x,"
280
"Tensor! conv_state,"
281
"Tensor! weight,"
282
- "Tensor? bias_,"
283
- "bool silu_activation) -> Tensor");
282
+ "Tensor? bias,"
283
+ "bool silu_activation,"
284
+ "Tensor? conv_state_indices) -> Tensor");
285
ops.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update);
286
287
ops.def(
tests/kernels/test_causal_conv1d.py
+58
@@ -203,3 +203,61 @@ def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
203
204
assert torch.equal(conv_state, conv_state_ref)
205
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
206
+
207
+
208
+@pytest.mark.parametrize("itype",
209
+ [torch.float32, torch.float16, torch.bfloat16])
210
+@pytest.mark.parametrize("silu_activation", [False, True])
211
+@pytest.mark.parametrize("has_bias", [False, True])
212
+@pytest.mark.parametrize("seqlen", [1, 4, 5])
213
+@pytest.mark.parametrize("width", [2, 3, 4])
214
+@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
215
+def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
216
+ silu_activation, itype):
217
+ device = "cuda"
218
+ rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
219
+ if itype == torch.bfloat16:
220
+ rtol, atol = 1e-2, 5e-2
221
+
222
+ # set seed
223
+ torch.random.manual_seed(0)
224
+ batch = 64
225
+
226
+ x = torch.randn(batch, dim, device=device, dtype=itype)
227
+
228
+ total_entries = 10 * batch
229
+ conv_state = torch.randn(total_entries,
230
+ dim,
231
+ width,
232
+ device=device,
233
+ dtype=itype)
234
+ conv_state_indices = torch.randperm(total_entries)[:batch].to(
235
+ dtype=torch.int32, device=device)
236
+
237
+ weight = torch.randn(dim,
238
+ width,
239
+ device=device,
240
+ dtype=itype,
241
+ requires_grad=True)
242
+ if has_bias:
243
+ bias = torch.randn(dim, device=device, dtype=itype, requires_grad=True)
244
+ else:
245
+ bias = None
246
+ conv_state_ref = conv_state[conv_state_indices, :].detach().clone()
247
+ activation = None if not silu_activation else "silu"
248
+ out = causal_conv1d_update(x,
249
+ conv_state,
250
+ weight,
251
+ bias,
252
+ activation=activation,
253
+ conv_state_indices=conv_state_indices)
254
+ out_ref = causal_conv1d_update_ref(x,
255
+ conv_state_ref,
256
+ weight,
257
+ bias,
258
+ activation=activation)
259
+
260
+ print(f"Output max diff: {(out - out_ref).abs().max().item()}")
261
+ print(f"Output mean diff: {(out - out_ref).abs().mean().item()}")
262
+ assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
263
+ assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
vllm/_custom_ops.py
+10
-4
@@ -768,11 +768,17 @@ def causal_conv1d_fwd(x: torch.Tensor, weight: torch.Tensor,
768
silu_activation)
769
770
771
-def causal_conv1d_update(x: torch.Tensor, conv_state: torch.Tensor,
772
- weight: torch.Tensor, bias_: Optional[torch.Tensor],
773
- silu_activation: bool) -> torch.Tensor:
771
+def causal_conv1d_update(
772
+ x: torch.Tensor,
773
+ conv_state: torch.Tensor,
774
+ weight: torch.Tensor,
775
+ bias_: Optional[torch.Tensor],
776
+ silu_activation: bool,
777
+ conv_state_indices: Optional[torch.Tensor],
778
+) -> torch.Tensor:
779
return torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
775
- silu_activation)
780
+ silu_activation,
781
+ conv_state_indices)
782
783
784
def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
vllm/model_executor/layers/mamba/ops/causal_conv1d.py
+8
-2
@@ -1,4 +1,5 @@
1
# Copyright (c) 2024, Tri Dao.
2
+# Adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/causal_conv1d/causal_conv1d_interface.py
3
4
from typing import Optional
5
@@ -70,12 +71,17 @@ def causal_conv1d_update(x: torch.Tensor,
71
conv_state: torch.Tensor,
72
weight: torch.Tensor,
73
bias: Optional[torch.Tensor] = None,
73
- activation: Optional[str] = None):
74
+ activation: Optional[str] = None,
75
+ conv_state_indices: Optional[torch.Tensor] = None):
76
"""
77
x: (batch, dim)
78
conv_state: (batch, dim, width)
79
weight: (dim, width)
80
bias: (dim,)
81
+ conv_state_indices: (batch,), dtype int32
82
+ If not None, the conv_state is a larger tensor along the batch dim,
83
+ and we are selecting the batch coords specified by conv_state_indices.
84
+ Useful for a continuous batching scenario.
85
86
out: (batch, dim)
87
"""
@@ -83,4 +89,4 @@ def causal_conv1d_update(x: torch.Tensor,
89
raise NotImplementedError("activation must be None, silu, or swish")
90
activation_bool = activation in ["silu", "swish"]
91
return ops.causal_conv1d_update(x, conv_state, weight, bias,
86
- activation_bool)
92
+ activation_bool, conv_state_indices)