A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Kernel][Model] Improve continuous batching for Jamba and Mamba (#9189)
Mor Zusman committed
Oct 17, 2024 at 00:12 UTC
fb60ae9b91a4b3e1aed4a6e826895fe3c5a13c10
15 files changed
+511
-439
csrc/mamba/causal_conv1d/causal_conv1d.cu
+25
-12
@@ -55,6 +55,7 @@ void set_conv_params_fwd(ConvParamsBase ¶ms,
55
const at::Tensor out,
56
const c10::optional<at::Tensor>& bias,
57
bool silu_activation,
58
+ int64_t pad_slot_id,
59
const c10::optional<at::Tensor>& query_start_loc = std::nullopt,
60
const c10::optional<at::Tensor>& cache_indices = std::nullopt,
61
const c10::optional<at::Tensor>& has_initial_state = std::nullopt) {
@@ -66,6 +67,7 @@ void set_conv_params_fwd(ConvParamsBase ¶ms,
67
params.dim = dim;
68
params.seqlen = seqlen;
69
params.width = width;
70
+ params.pad_slot_id = pad_slot_id;
71
72
params.silu_activation = silu_activation;
73
@@ -90,14 +92,16 @@ void set_conv_params_fwd(ConvParamsBase ¶ms,
92
}
93
94
93
-at::Tensor
94
-causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
95
+void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
96
const c10::optional<at::Tensor> &bias_,
97
const c10::optional<at::Tensor> &conv_states,
98
const c10::optional<at::Tensor> &query_start_loc,
99
const c10::optional<at::Tensor> &cache_indices,
100
const c10::optional<at::Tensor> &has_initial_state,
100
- bool silu_activation) {
101
+ bool silu_activation,
102
+ // used to identify padding entries if cache_indices provided
103
+ // in case of padding, the kernel will return early
104
+ int64_t pad_slot_id) {
105
auto input_type = x.scalar_type();
106
auto weight_type = weight.scalar_type();
107
TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -153,12 +157,13 @@ causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
157
CHECK_SHAPE(cache_indices_, batch_size);
158
}
159
156
- at::Tensor out = torch::empty_like(x);
160
+ at::Tensor out = x;
161
162
ConvParamsBase params;
163
set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
164
bias_,
165
silu_activation,
166
+ pad_slot_id,
167
query_start_loc,
168
cache_indices,
169
has_initial_state
@@ -183,18 +188,19 @@ causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
188
DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_fwd", [&] {
189
causal_conv1d_fwd_cuda<input_t, weight_t>(params, stream);
190
});
186
- return out;
191
}
192
193
190
-at::Tensor
191
-causal_conv1d_update(const at::Tensor &x,
194
+void causal_conv1d_update(const at::Tensor &x,
195
const at::Tensor &conv_state,
196
const at::Tensor &weight,
197
const c10::optional<at::Tensor> &bias_,
198
bool silu_activation,
199
const c10::optional<at::Tensor> &cache_seqlens_,
197
- const c10::optional<at::Tensor> &conv_state_indices_) {
200
+ const c10::optional<at::Tensor> &conv_state_indices_,
201
+ // used to identify padding entries if cache_indices provided
202
+ // in case of padding, the kernel will return early
203
+ int64_t pad_slot_id) {
204
auto input_type = x.scalar_type();
205
auto weight_type = weight.scalar_type();
206
TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -227,12 +233,13 @@ causal_conv1d_update(const at::Tensor &x,
233
CHECK_SHAPE(bias, dim);
234
}
235
230
- at::Tensor out = torch::empty_like(x);
236
+ at::Tensor out = x;
237
238
ConvParamsBase params;
239
set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
240
bias_,
235
- silu_activation);
241
+ silu_activation,
242
+ pad_slot_id);
243
params.conv_state_ptr = conv_state.data_ptr();
244
params.conv_state_len = conv_state_len;
245
// All stride are in elements, not bytes.
@@ -274,7 +281,6 @@ causal_conv1d_update(const at::Tensor &x,
281
DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_update", [&] {
282
causal_conv1d_update_cuda<input_t, weight_t>(params, stream);
283
});
277
- return out;
284
}
285
286
template<int kNThreads_, int kWidth_, bool kIsVecLoad_, typename input_t_, typename weight_t_>
@@ -340,7 +346,10 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
346
int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr
347
: reinterpret_cast<int *>(params.cache_indices_ptr);
348
int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id];
343
-
349
+ // cache_index == params.pad_slot_id is defined as padding, so we exit early
350
+ if (cache_index == params.pad_slot_id){
351
+ return;
352
+ }
353
input_t *conv_states = params.conv_states_ptr == nullptr ? nullptr
354
: reinterpret_cast<input_t *>(params.conv_states_ptr) + cache_index * params.conv_states_batch_stride + channel_id * params.conv_states_c_stride;
355
@@ -528,6 +537,10 @@ void causal_conv1d_update_kernel(ConvParamsBase params) {
537
const int conv_state_batch_coord = params.conv_state_indices_ptr == nullptr
538
? batch_id
539
: params.conv_state_indices_ptr[batch_id];
540
+ // conv_state_batch_coord == params.pad_slot_id is defined as padding so we exit early
541
+ if (conv_state_batch_coord == params.pad_slot_id){
542
+ return;
543
+ }
544
input_t *conv_state = reinterpret_cast<input_t *>(params.conv_state_ptr)
545
+ conv_state_batch_coord * params.conv_state_batch_stride
546
+ channel_id * params.conv_state_c_stride;
csrc/mamba/causal_conv1d/causal_conv1d.h
+1
@@ -13,6 +13,7 @@ struct ConvParamsBase {
13
using index_t = uint32_t;
14
15
int batch, dim, seqlen, width;
16
+ int64_t pad_slot_id;
17
bool silu_activation;
18
19
index_t x_batch_stride;
csrc/mamba/mamba_ssm/selective_scan.h
+1
@@ -21,6 +21,7 @@ struct SSMParamsBase {
21
int dim_ngroups_ratio;
22
bool is_variable_B;
23
bool is_variable_C;
24
+ int64_t pad_slot_id;
25
26
bool delta_softplus;
27
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
+14
-10
@@ -115,6 +115,10 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
115
const int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr
116
: reinterpret_cast<int *>(params.cache_indices_ptr);
117
const int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id];
118
+ // cache_index == params.pad_slot_id is defined as padding, so we exit early
119
+ if (cache_index == params.pad_slot_id){
120
+ return;
121
+ }
122
input_t *u = reinterpret_cast<input_t *>(params.u_ptr) + sequence_start_index * params.u_batch_stride
123
+ dim_id * kNRows * params.u_d_stride;
124
input_t *delta = reinterpret_cast<input_t *>(params.delta_ptr) + sequence_start_index * params.delta_batch_stride
@@ -387,7 +391,6 @@ void set_ssm_params_fwd(SSMParamsBase ¶ms,
391
const size_t seqlen,
392
const size_t dstate,
393
const size_t n_groups,
390
- const size_t n_chunks,
394
const bool is_variable_B,
395
const bool is_variable_C,
396
// device pointers
@@ -407,7 +410,8 @@ void set_ssm_params_fwd(SSMParamsBase ¶ms,
410
const c10::optional<at::Tensor>& query_start_loc,
411
const c10::optional<at::Tensor>& cache_indices,
412
const c10::optional<at::Tensor>& has_initial_state,
410
- bool varlen) {
413
+ bool varlen,
414
+ int64_t pad_slot_id) {
415
416
// Reset the parameters
417
memset(¶ms, 0, sizeof(params));
@@ -417,8 +421,8 @@ void set_ssm_params_fwd(SSMParamsBase ¶ms,
421
params.seqlen = seqlen;
422
params.dstate = dstate;
423
params.n_groups = n_groups;
420
- params.n_chunks = n_chunks;
424
params.dim_ngroups_ratio = dim / n_groups;
425
+ params.pad_slot_id = pad_slot_id;
426
427
params.delta_softplus = delta_softplus;
428
@@ -507,7 +511,10 @@ void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
511
const c10::optional<torch::Tensor> &query_start_loc,
512
const c10::optional<torch::Tensor> &cache_indices,
513
const c10::optional<torch::Tensor> &has_initial_state,
510
- const torch::Tensor &ssm_states) {
514
+ const torch::Tensor &ssm_states,
515
+ // used to identify padding entries if cache_indices provided
516
+ // in case of padding, the kernel will return early
517
+ int64_t pad_slot_id) {
518
auto input_type = u.scalar_type();
519
auto weight_type = A.scalar_type();
520
TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -618,18 +625,14 @@ void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
625
626
out_z = z;
627
621
- const int n_chunks = (seqlen + 2048 - 1) / 2048;
622
- // const int n_chunks = (seqlen + 1024 - 1) / 1024;
623
- // at::Tensor out = torch::empty_like(u);
628
// Right now u has BHL layout and delta has HBL layout, and we want out to have HBL layout
629
at::Tensor out = delta;
630
TORCH_CHECK(ssm_states.scalar_type() == input_type);
631
TORCH_CHECK(ssm_states.is_cuda());
632
TORCH_CHECK(ssm_states.stride(-1) == 1);
629
- CHECK_SHAPE(ssm_states, batch_size, dim, dstate);
633
634
SSMParamsBase params;
632
- set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, n_chunks, is_variable_B, is_variable_C,
635
+ set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, is_variable_B, is_variable_C,
636
u, delta, A, B, C, out, z, out_z,
637
D_,
638
delta_bias_,
@@ -639,7 +642,8 @@ void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
642
query_start_loc,
643
cache_indices,
644
has_initial_state,
642
- varlen
645
+ varlen,
646
+ pad_slot_id
647
);
648
649
csrc/ops.h
+17
-15
@@ -157,21 +157,23 @@ void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
157
const c10::optional<torch::Tensor>& query_start_loc,
158
const c10::optional<torch::Tensor>& cache_indices,
159
const c10::optional<torch::Tensor>& has_initial_state,
160
- const torch::Tensor& ssm_states);
161
-
162
-at::Tensor causal_conv1d_update(
163
- const at::Tensor& x, const at::Tensor& conv_state, const at::Tensor& weight,
164
- const c10::optional<at::Tensor>& bias_, bool silu_activation,
165
- const c10::optional<at::Tensor>& cache_seqlens_,
166
- const c10::optional<at::Tensor>& conv_state_indices_);
167
-
168
-at::Tensor causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
169
- const c10::optional<at::Tensor>& bias_,
170
- const c10::optional<at::Tensor>& conv_states,
171
- const c10::optional<at::Tensor>& query_start_loc,
172
- const c10::optional<at::Tensor>& cache_indices,
173
- const c10::optional<at::Tensor>& has_initial_state,
174
- bool silu_activation);
160
+ const torch::Tensor& ssm_states, int64_t pad_slot_id);
161
+
162
+void causal_conv1d_update(const at::Tensor& x, const at::Tensor& conv_state,
163
+ const at::Tensor& weight,
164
+ const c10::optional<at::Tensor>& bias_,
165
+ bool silu_activation,
166
+ const c10::optional<at::Tensor>& cache_seqlens_,
167
+ const c10::optional<at::Tensor>& conv_state_indices_,
168
+ int64_t pad_slot_id);
169
+
170
+void causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
171
+ const c10::optional<at::Tensor>& bias_,
172
+ const c10::optional<at::Tensor>& conv_states,
173
+ const c10::optional<at::Tensor>& query_start_loc,
174
+ const c10::optional<at::Tensor>& cache_indices,
175
+ const c10::optional<at::Tensor>& has_initial_state,
176
+ bool silu_activation, int64_t pad_slot_id);
177
178
#ifndef USE_ROCM
179
using fptr_t = int64_t;
csrc/torch_bindings.cpp
+6
-3
@@ -278,7 +278,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
278
"Tensor? query_start_loc,"
279
"Tensor? cache_indices,"
280
"Tensor? has_initial_state,"
281
- "Tensor! ssm_states) -> ()");
281
+ "Tensor! ssm_states,"
282
+ "int pad_slot_id) -> ()");
283
ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
284
285
ops.def(
@@ -288,7 +289,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
289
"Tensor? bias_,"
290
"bool silu_activation,"
291
"Tensor? cache_seqlens_,"
291
- "Tensor? conv_state_indices) -> Tensor");
292
+ "Tensor? conv_state_indices,"
293
+ "int pad_slot_id) -> ()");
294
ops.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update);
295
296
ops.def(
@@ -298,7 +300,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
300
"Tensor? query_start_loc,"
301
"Tensor? cache_indices,"
302
"Tensor? has_initial_state,"
301
- "bool silu_activation) -> Tensor");
303
+ "bool silu_activation,"
304
+ "int pad_slot_id) -> ()");
305
ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
306
#endif
307
tests/kernels/test_causal_conv1d.py
+97
-94
@@ -6,6 +6,7 @@ import torch.nn.functional as F
6
7
from tests.kernels.utils import opcheck
8
from vllm import _custom_ops as ops # noqa: F401
9
+from vllm.attention.backends.utils import PAD_SLOT_ID
10
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
11
causal_conv1d_fn, causal_conv1d_update)
12
from vllm.utils import seed_everything
@@ -114,16 +115,15 @@ def causal_conv1d_update_ref(x,
115
@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
116
@pytest.mark.parametrize("silu_activation", [True])
117
@pytest.mark.parametrize("has_bias", [True])
117
-def causal_conv1d_opcheck_fn(
118
- x: torch.Tensor,
119
- weight: torch.Tensor,
120
- bias: Optional[torch.Tensor] = None,
121
- cu_seq_len: Optional[torch.Tensor] = None,
122
- cache_indices: Optional[torch.Tensor] = None,
123
- has_initial_state: Optional[torch.Tensor] = None,
124
- conv_states: Optional[torch.Tensor] = None,
125
- activation: Optional[str] = "silu",
126
-):
118
+def causal_conv1d_opcheck_fn(x: torch.Tensor,
119
+ weight: torch.Tensor,
120
+ bias: Optional[torch.Tensor] = None,
121
+ cu_seq_len: Optional[torch.Tensor] = None,
122
+ cache_indices: Optional[torch.Tensor] = None,
123
+ has_initial_state: Optional[torch.Tensor] = None,
124
+ conv_states: Optional[torch.Tensor] = None,
125
+ activation: Optional[str] = "silu",
126
+ pad_slot_id: int = PAD_SLOT_ID):
127
"""
128
x: (batch, dim, seqlen)
129
weight: (dim, width)
@@ -141,16 +141,9 @@ def causal_conv1d_opcheck_fn(
141
x = x.contiguous()
142
bias = bias.contiguous() if bias is not None else None
143
144
- opcheck(torch.ops._C.causal_conv1d_fwd, (
145
- x,
146
- weight,
147
- bias,
148
- conv_states,
149
- cu_seq_len,
150
- cache_indices,
151
- has_initial_state,
152
- activation in ["silu", "swish"],
153
- ))
144
+ opcheck(torch.ops._C.causal_conv1d_fwd,
145
+ (x, weight, bias, conv_states, cu_seq_len, cache_indices,
146
+ has_initial_state, activation in ["silu", "swish"], pad_slot_id))
147
148
149
@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
@@ -233,17 +226,11 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
226
seed_everything(0)
227
batch = 2
228
x = torch.randn(batch, dim, seqlen, device=device, dtype=itype)
229
+ x_ref = x.clone()
230
conv_state = torch.randn(batch, dim, width - 1, device=device, dtype=itype)
231
238
- weight = torch.randn(dim,
239
- width,
240
- device=device,
241
- dtype=itype,
242
- requires_grad=True)
243
- if has_bias:
244
- bias = torch.randn(dim, device=device, dtype=itype, requires_grad=True)
245
- else:
246
- bias = None
232
+ weight = torch.randn(dim, width, device=device, dtype=itype)
233
+ bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
234
conv_state_ref = conv_state.detach().clone()
235
activation = None if not silu_activation else "silu"
236
out = causal_conv1d_update(x,
@@ -251,7 +238,7 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
238
weight,
239
bias,
240
activation=activation)
254
- out_ref = causal_conv1d_update_ref(x,
241
+ out_ref = causal_conv1d_update_ref(x_ref,
242
conv_state_ref,
243
weight,
244
bias,
@@ -260,15 +247,9 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
247
assert torch.equal(conv_state, conv_state_ref)
248
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
249
263
- opcheck(torch.ops._C.causal_conv1d_update, (
264
- x,
265
- conv_state,
266
- weight,
267
- bias,
268
- activation in ["silu", "swish"],
269
- None,
270
- None,
271
- ))
250
+ opcheck(torch.ops._C.causal_conv1d_update,
251
+ (x, conv_state, weight, bias, activation
252
+ in ["silu", "swish"], None, None, PAD_SLOT_ID))
253
254
255
@pytest.mark.parametrize("itype",
@@ -278,37 +259,48 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
259
@pytest.mark.parametrize("seqlen", [1, 4, 5])
260
@pytest.mark.parametrize("width", [2, 3, 4])
261
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
281
-def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
262
+# tests correctness in case subset of the sequences are padded
263
+@pytest.mark.parametrize("with_padding", [True, False])
264
+def test_causal_conv1d_update_with_batch_gather(with_padding, dim, width,
265
+ seqlen, has_bias,
266
silu_activation, itype):
267
device = "cuda"
268
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
269
if itype == torch.bfloat16:
270
rtol, atol = 1e-2, 5e-2
271
288
- # set )seed
272
+ # set seed
273
seed_everything(0)
290
- batch = 64
274
292
- x = torch.randn(batch, dim, 1, device=device, dtype=itype)
275
+ batch_size = 3
276
+ padding = 5 if with_padding else 0
277
+ padded_batch_size = batch_size + padding
278
+ total_entries = 10 * batch_size
279
294
- total_entries = 10 * batch
280
+ x = torch.randn(padded_batch_size, dim, 1, device=device, dtype=itype)
281
+ x_ref = x.clone()
282
+
283
+ conv_state_indices = torch.randperm(total_entries)[:batch_size].to(
284
+ dtype=torch.int32, device=device)
285
+ unused_states_bool = torch.ones(total_entries,
286
+ dtype=torch.bool,
287
+ device=device)
288
+ unused_states_bool[conv_state_indices] = False
289
+ padded_state_indices = torch.concat([
290
+ conv_state_indices,
291
+ torch.as_tensor(
292
+ [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device)
293
+ ],
294
+ dim=0)
295
conv_state = torch.randn(total_entries,
296
dim,
297
width - 1,
298
device=device,
299
dtype=itype)
300
- conv_state_indices = torch.randperm(total_entries)[:batch].to(
301
- dtype=torch.int32, device=device)
300
+ conv_state_for_padding_test = conv_state.clone()
301
303
- weight = torch.randn(dim,
304
- width,
305
- device=device,
306
- dtype=itype,
307
- requires_grad=True)
308
- if has_bias:
309
- bias = torch.randn(dim, device=device, dtype=itype, requires_grad=True)
310
- else:
311
- bias = None
302
+ weight = torch.randn(dim, width, device=device, dtype=itype)
303
+ bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
304
conv_state_ref = conv_state[conv_state_indices, :].detach().clone()
305
activation = None if not silu_activation else "silu"
306
out = causal_conv1d_update(x,
@@ -316,45 +308,50 @@ def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
308
weight,
309
bias,
310
activation=activation,
319
- conv_state_indices=conv_state_indices)
320
- out_ref = causal_conv1d_update_ref(x,
311
+ conv_state_indices=padded_state_indices,
312
+ pad_slot_id=PAD_SLOT_ID)
313
+ out_ref = causal_conv1d_update_ref(x_ref[:batch_size],
314
conv_state_ref,
315
weight,
316
bias,
317
activation=activation)
318
319
assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
327
- assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
320
+ assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol)
321
+ assert torch.equal(conv_state[unused_states_bool],
322
+ conv_state_for_padding_test[unused_states_bool])
323
329
- opcheck(torch.ops._C.causal_conv1d_update, (
330
- x,
331
- conv_state,
332
- weight,
333
- bias,
334
- activation in ["silu", "swish"],
335
- None,
336
- conv_state_indices,
337
- ))
324
+ opcheck(torch.ops._C.causal_conv1d_update,
325
+ (x, conv_state, weight, bias, activation
326
+ in ["silu", "swish"], None, padded_state_indices, PAD_SLOT_ID))
327
328
329
@pytest.mark.parametrize("itype", [torch.bfloat16])
330
@pytest.mark.parametrize("silu_activation", [True])
331
@pytest.mark.parametrize("has_bias", [True])
332
@pytest.mark.parametrize("width", [4])
344
-@pytest.mark.parametrize('seqlen',
345
- [8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 4096])
333
+@pytest.mark.parametrize(
334
+ 'seqlen', [8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 2049, 4096])
335
@pytest.mark.parametrize('dim', [64, 4096])
347
-def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
348
- itype):
336
+# tests correctness in case subset of the sequences are padded
337
+@pytest.mark.parametrize('with_padding', [True, False])
338
+def test_causal_conv1d_varlen(with_padding, dim, seqlen, width, has_bias,
339
+ silu_activation, itype):
340
device = "cuda"
341
+ torch.cuda.empty_cache()
342
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
343
if itype == torch.bfloat16:
344
rtol, atol = 1e-2, 5e-2
345
# set seed
346
seed_everything(0)
355
- batch = 1
347
seqlens = []
357
- nsplits = 3
348
+ batch_size = 4
349
+ if seqlen < 10:
350
+ batch_size = 1
351
+ padding = 3 if with_padding else 0
352
+ padded_batch_size = batch_size + padding
353
+ nsplits = padded_batch_size - 1
354
+
355
eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
356
seqlens.append(
357
torch.diff(
@@ -364,10 +361,11 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
361
assert sum(seqlens[-1]) == seqlen
362
assert all(s > 0 for s in seqlens[-1])
363
364
+ total_entries = batch_size * 10
365
cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
366
cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum],
367
dim=0)
370
- x = torch.randn(batch, 4096 + dim + 64, seqlen, device=device,
368
+ x = torch.randn(1, 4096 + dim + 64, seqlen, device=device,
369
dtype=itype)[:, 4096:4096 + dim, :]
370
weight = torch.randn(dim, width, device=device, dtype=itype)
371
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
@@ -375,7 +373,7 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
373
weight_ref = weight.clone()
374
bias_ref = bias.clone() if bias is not None else None
375
activation = None if not silu_activation else "silu"
378
- final_states = torch.randn(nsplits + 1,
376
+ final_states = torch.randn(total_entries,
377
dim,
378
width - 1,
379
device=x.device,
@@ -385,18 +383,27 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
383
2, (cumsum.shape[0] - 1, ),
384
dtype=torch.bool,
385
device=x.device)
388
- cache_indices = torch.randperm(cumsum.shape[0] - 1,
386
+ state_indices = torch.randperm(total_entries,
387
dtype=torch.int32,
390
- device=x.device)
388
+ device=x.device)[:batch_size]
389
+ padded_state_indices = torch.concat([
390
+ state_indices,
391
+ torch.as_tensor(
392
+ [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device),
393
+ ],
394
+ dim=-1)
395
+
396
out = causal_conv1d_fn(x.squeeze(0), weight, bias, cumsum.cuda(),
392
- cache_indices, has_initial_states, final_states,
393
- activation)
397
+ padded_state_indices, has_initial_states,
398
+ final_states, activation, PAD_SLOT_ID)
399
out_ref = []
400
out_ref_b = []
401
402
splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)]
403
for i in range(len(seqlens[0])):
404
x_s = [v[i].unsqueeze(0) for v in splits][0]
405
+ if padded_state_indices[i] == PAD_SLOT_ID:
406
+ continue
407
out_ref_b.append(
408
causal_conv1d_ref(
409
x_s,
@@ -404,21 +411,17 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
411
bias_ref,
412
activation=activation,
413
return_final_states=True,
407
- final_states_out=final_states_ref[cache_indices[i]].unsqueeze(
408
- 0),
409
- initial_states=final_states_ref[cache_indices[i]].unsqueeze(0)
410
- if has_initial_states[i] else None))
414
+ final_states_out=final_states_ref[
415
+ padded_state_indices[i]].unsqueeze(0),
416
+ initial_states=final_states_ref[padded_state_indices[i]].
417
+ unsqueeze(0) if has_initial_states[i] else None))
418
out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2))
412
- out_ref = torch.cat(out_ref, dim=0)
413
-
414
- print(f"Output max diff: {(out - out_ref).abs().max().item()}")
415
- print(f"Output mean diff: {(out - out_ref).abs().mean().item()}")
416
- print("Output state max diff"
417
- f":{(final_states - final_states_ref).abs().max()}")
418
- print("Output state mean diff"
419
- f":{(final_states - final_states_ref).abs().mean()}")
420
- assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
419
+ out_ref_tensor = torch.cat(out_ref, dim=0)
420
+
421
+ unpadded_out = out[:, :out_ref_tensor.shape[-1]]
422
+ assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
423
assert torch.allclose(final_states, final_states_ref, rtol=rtol, atol=atol)
424
+
425
causal_conv1d_opcheck_fn(x.squeeze(0), weight, bias, cumsum.cuda(),
423
- cache_indices, has_initial_states, final_states,
424
- activation)
426
+ padded_state_indices, has_initial_states,
427
+ final_states, activation)
tests/kernels/test_mamba_ssm.py
+89
-35
@@ -5,6 +5,7 @@ from einops import rearrange, repeat
5
6
from tests.kernels.utils import opcheck
7
from vllm import _custom_ops as ops # noqa: F401
8
+from vllm.attention.backends.utils import PAD_SLOT_ID
9
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
10
selective_scan_fn, selective_state_update)
11
from vllm.utils import seed_everything
@@ -174,7 +175,8 @@ def selective_scan_opcheck_fn(u,
175
cu_seq_len=None,
176
cache_indices=None,
177
has_initial_state=None,
177
- ssm_states=None):
178
+ ssm_states=None,
179
+ pad_slot_id=PAD_SLOT_ID):
180
"""if return_last_state is True, returns (out, last_state)
181
last_state has shape (batch, dim, dstate).
182
"""
@@ -203,7 +205,7 @@ def selective_scan_opcheck_fn(u,
205
# a bogus error.
206
opcheck(torch.ops._C.selective_scan_fwd,
207
(u, delta, A, B, C, D, z, delta_bias, delta_softplus, cu_seq_len,
206
- cache_indices, has_initial_state, ssm_states),
208
+ cache_indices, has_initial_state, ssm_states, pad_slot_id),
209
test_utils=["test_schema", "test_faketensor"])
210
211
@@ -404,9 +406,12 @@ def test_selective_state_update(dim, dstate, has_z, itype):
406
@pytest.mark.parametrize("varBC_groups", [1, 2])
407
@pytest.mark.parametrize("is_variable_C", [True])
408
@pytest.mark.parametrize("is_variable_B", [True])
407
-def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
408
- has_D, has_z, has_delta_bias, delta_softplus,
409
- return_last_state, seqlen, itype, wtype):
409
+# tests correctness in case subset of the sequences are padded
410
+@pytest.mark.parametrize("with_padding", [False, True])
411
+def test_selective_scan_varlen(with_padding, is_variable_B, is_variable_C,
412
+ varBC_groups, has_D, has_z, has_delta_bias,
413
+ delta_softplus, return_last_state, seqlen,
414
+ itype, wtype):
415
if varBC_groups > 1 and (not is_variable_B or not is_variable_C):
416
pytest.skip() # This config is not applicable
417
device = 'cuda'
@@ -420,18 +425,27 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
425
# set seed
426
torch.random.manual_seed(0)
427
seqlens = []
423
- nsplits = 3
428
+ batch_size = 4
429
if seqlen < 10:
425
- nsplits = 0
430
+ batch_size = 1
431
+ padding = 3 if with_padding else 0
432
+ padded_batch_size = batch_size + padding
433
+
434
+ if with_padding and seqlen < padded_batch_size:
435
+ pytest.skip()
436
+
437
+ nsplits = padded_batch_size - 1
438
eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
439
seqlens.append(
440
torch.diff(
441
torch.cat(
442
[torch.tensor([-1]), eos_pos,
443
torch.tensor([seqlen - 1])])).tolist())
444
+
445
assert sum(seqlens[-1]) == seqlen
446
assert all(s > 0 for s in seqlens[-1])
447
448
+ total_entries = batch_size * 10
449
cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
450
cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum],
451
dim=0).cuda()
@@ -462,22 +476,33 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
476
delta_ref = delta.clone()
477
out = None
478
out_ref = None
465
- prev_state_shape = (cumsum.shape[0] - 1, u.shape[0], int(A.shape[1]))
479
+
480
+ prev_state_shape = (total_entries, u.shape[0], int(A.shape[1]))
481
prev_state = torch.randn(prev_state_shape,
482
device=u.device,
483
dtype=itype,
484
requires_grad=False)
485
prev_state_ref = prev_state.clone()
471
- cache_indices = torch.randperm(cumsum.shape[0] - 1,
486
+ state_indices = torch.randperm(total_entries,
487
dtype=torch.int32,
473
- device=u.device)
488
+ device=u.device)[:batch_size]
489
+ unused_states_bool = torch.ones(total_entries,
490
+ dtype=torch.bool,
491
+ device=device)
492
+ unused_states_bool[state_indices] = False
493
+ padded_state_indices = torch.concat([
494
+ state_indices,
495
+ torch.as_tensor(
496
+ [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device),
497
+ ],
498
+ dim=-1)
499
500
has_initial_state = torch.randint(0,
501
2, (cumsum.shape[0] - 1, ),
502
dtype=torch.bool,
503
device=u.device)
504
out = selective_scan_fn(u, prev_state, delta, A, B, C, D, z, delta_bias,
480
- delta_softplus, cumsum, cache_indices,
505
+ delta_softplus, cumsum, padded_state_indices,
506
has_initial_state)
507
outs_ref = []
508
splits = [
@@ -486,6 +511,8 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
511
]
512
for i in range(len(seqlens[0])):
513
u_s, delta_s, B_s, C_s, z_s = [v[i].unsqueeze(0) for v in splits]
514
+ if padded_state_indices[i] == PAD_SLOT_ID:
515
+ continue
516
out_ref_s, _ = selective_scan_ref(
517
u_s,
518
delta_s,
@@ -497,21 +524,22 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
524
delta_bias=delta_bias,
525
delta_softplus=delta_softplus,
526
return_last_state=return_last_state,
500
- prev_state=prev_state_ref[cache_indices[i]].unsqueeze(0)
527
+ prev_state=prev_state_ref[padded_state_indices[i]].unsqueeze(0)
528
if has_initial_state[i] else None,
502
- final_state_out=prev_state_ref[cache_indices[i]].unsqueeze(0))
529
+ final_state_out=prev_state_ref[padded_state_indices[i]].unsqueeze(
530
+ 0))
531
outs_ref.append(out_ref_s)
504
- out_ref = torch.cat(outs_ref, dim=-1) if len(outs_ref) > 1 else outs_ref[0]
532
+ out_ref = torch.cat(outs_ref, dim=-1)[0]
533
506
- print("Output diff max", (out - out_ref[0]).max())
507
- print("Output diff mean", (out - out_ref[0]).mean())
534
+ unpadded_out = out[:, :out_ref[0].shape[-1]]
535
+ print("Output diff max", (unpadded_out - out_ref).max())
536
+ print("Output diff mean", (unpadded_out - out_ref).mean())
537
print("Output state diff max", (prev_state - prev_state_ref).max())
538
print("Output state diff mean", (prev_state - prev_state_ref).mean())
539
assert torch.allclose(prev_state, prev_state_ref, rtol=rtol, atol=atol)
511
- assert torch.allclose(out, out_ref[0], rtol=rtol, atol=atol)
512
-
540
+ assert torch.allclose(unpadded_out, out_ref, rtol=rtol, atol=atol)
541
selective_scan_opcheck_fn(u, delta, A, B, C, D, z, delta_bias,
514
- delta_softplus, cumsum, cache_indices,
542
+ delta_softplus, cumsum, padded_state_indices,
543
has_initial_state, prev_state)
544
545
@@ -520,7 +548,10 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
548
@pytest.mark.parametrize("has_z", [True])
549
@pytest.mark.parametrize("dstate", [16, 32, 64])
550
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
523
-def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
551
+# tests correctness in case subset of the sequences are padded
552
+@pytest.mark.parametrize("with_padding", [True, False])
553
+def test_selective_state_update_with_batch_indices(with_padding, dim, dstate,
554
+ has_z, itype):
555
device = "cuda"
556
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
557
if itype == torch.bfloat16:
@@ -530,21 +561,32 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
561
# set seed
562
torch.random.manual_seed(0)
563
batch_size = 3
533
-
564
+ padding = 5 if with_padding else 0
565
+ padded_batch_size = batch_size + padding
566
total_entries = 10 * batch_size
567
state = torch.randn(total_entries, dim, dstate, dtype=itype, device=device)
568
state_indices = torch.randperm(total_entries)[:batch_size].to(
569
dtype=torch.int32, device=device)
538
-
539
- x = torch.randn(batch_size, dim, device=device, dtype=itype)
540
- dt = torch.randn(batch_size, dim, device=device, dtype=itype)
570
+ unused_states_bool = torch.ones(total_entries,
571
+ dtype=torch.bool,
572
+ device=device)
573
+ unused_states_bool[state_indices] = False
574
+ padded_state_indices = torch.concat([
575
+ state_indices,
576
+ torch.as_tensor(
577
+ [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device)
578
+ ],
579
+ dim=0)
580
+ x = torch.randn(padded_batch_size, dim, device=device, dtype=itype)
581
+ dt = torch.randn(padded_batch_size, dim, device=device, dtype=itype)
582
dt_bias = torch.rand(dim, device=device) - 4.0
583
A = -torch.rand(dim, dstate, device=device) - 1.0
543
- B = torch.randn(batch_size, dstate, device=device)
544
- C = torch.randn(batch_size, dstate, device=device)
584
+ B = torch.randn(padded_batch_size, dstate, device=device)
585
+ C = torch.randn(padded_batch_size, dstate, device=device)
586
D = torch.randn(dim, device=device)
587
z = torch.randn_like(x) if has_z else None
547
- state_ref = state[state_indices, :].detach().clone()
588
+ state_ref = state[state_indices, :].clone()
589
+ state_before = state.clone()
590
out = selective_state_update(state,
591
x,
592
dt,
@@ -555,15 +597,16 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
597
z=z,
598
dt_bias=dt_bias,
599
dt_softplus=True,
558
- state_batch_indices=state_indices)
600
+ state_batch_indices=padded_state_indices,
601
+ pad_slot_id=PAD_SLOT_ID)
602
out_ref = selective_state_update_ref(state_ref,
560
- x,
561
- dt,
603
+ x[:batch_size],
604
+ dt[:batch_size],
605
A,
563
- B,
564
- C,
606
+ B[:batch_size],
607
+ C[:batch_size],
608
D=D,
566
- z=z,
609
+ z=z[:batch_size],
610
dt_bias=dt_bias,
611
dt_softplus=True)
612
@@ -572,11 +615,21 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
615
print("Output state diff max", (state[state_indices, :] - state_ref).max())
616
print("Output state diff mean",
617
(state[state_indices, :] - state_ref).mean())
618
+ # test padded entries stay the same
619
+ if with_padding:
620
+ assert torch.equal(state_before[unused_states_bool],
621
+ state[unused_states_bool])
622
+ assert torch.equal(x[batch_size + 1:], x[batch_size + 1:])
623
+ assert torch.equal(dt[batch_size + 1:], dt[batch_size + 1:])
624
+ assert torch.equal(B[batch_size + 1:], B[batch_size + 1:])
625
+ assert torch.equal(C[batch_size + 1:], C[batch_size + 1:])
626
+
627
+ # test "real" entries
628
assert torch.allclose(state[state_indices, :],
629
state_ref,
630
rtol=rtol,
631
atol=atol)
579
- assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
632
+ assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol)
633
634
635
@pytest.mark.parametrize("itype",
@@ -645,7 +698,8 @@ def test_selective_state_update_with_heads_with_batch_indices(
698
z=z,
699
dt_bias=dt_bias,
700
dt_softplus=True,
648
- state_batch_indices=state_indices)
701
+ state_batch_indices=state_indices,
702
+ pad_slot_id=PAD_SLOT_ID)
703
out_ref = selective_state_update_ref(state_ref,
704
x,
705
dt,
tests/models/decoder_only/language/test_jamba.py
+25
@@ -1,5 +1,6 @@
1
import pytest
2
3
+from tests.utils import multi_gpu_test
4
from vllm.sampling_params import SamplingParams
5
from vllm.worker.model_runner import _get_graph_batch_size
6
@@ -270,6 +271,30 @@ def test_state_cleanup(
271
"could be related to finished_requests_ids")
272
273
274
+@multi_gpu_test(num_gpus=2)
275
+@pytest.mark.parametrize("model", MODELS)
276
+@pytest.mark.parametrize("dtype", ["float"])
277
+@pytest.mark.parametrize("max_tokens", [64])
278
+def test_jamba_distributed_produces_identical_generation(
279
+ vllm_runner, model: str, dtype: str, max_tokens: int,
280
+ example_prompts) -> None:
281
+
282
+ with vllm_runner(model, dtype=dtype, tensor_parallel_size=2) as vllm_model:
283
+ vllm_outputs_tp_2 = vllm_model.generate_greedy(example_prompts,
284
+ max_tokens)
285
+
286
+ with vllm_runner(model, dtype=dtype, tensor_parallel_size=1) as vllm_model:
287
+ vllm_outputs_tp_1 = vllm_model.generate_greedy(example_prompts,
288
+ max_tokens)
289
+
290
+ check_outputs_equal(
291
+ outputs_0_lst=vllm_outputs_tp_1,
292
+ outputs_1_lst=vllm_outputs_tp_2,
293
+ name_0="vllm_tp_1",
294
+ name_1="vllm_tp_2",
295
+ )
296
+
297
+
298
@pytest.mark.parametrize("model", MODELS)
299
@pytest.mark.parametrize("dtype", ["float"])
300
def test_model_print(
vllm/_custom_ops.py
+40
-33
@@ -464,16 +464,18 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
464
cu_seq_len: Optional[torch.Tensor],
465
cache_indices: Optional[torch.Tensor],
466
has_initial_state: Optional[torch.Tensor],
467
- silu_activation: bool) -> torch.Tensor:
468
- return torch.empty_like(x)
467
+ silu_activation: bool, pad_slot_id: int):
468
+ return None
469
470
@register_fake("_C::causal_conv1d_update")
471
- def causal_conv1d_update_fake(
472
- x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
473
- bias_: Optional[torch.Tensor], silu_activation: bool,
474
- cache_seqlens: Optional[torch.Tensor],
475
- conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
476
- return torch.empty_like(x)
471
+ def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
472
+ weight: torch.Tensor,
473
+ bias_: Optional[torch.Tensor],
474
+ silu_activation: bool,
475
+ cache_seqlens: Optional[torch.Tensor],
476
+ conv_state_indices: Optional[torch.Tensor],
477
+ pad_slot_id: int) -> None:
478
+ return None
479
480
@register_fake("_C::selective_scan_fwd")
481
def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor,
@@ -485,7 +487,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
487
cu_seq_len: Optional[torch.Tensor],
488
cache_indices: Optional[torch.Tensor],
489
has_initial_state: Optional[torch.Tensor],
488
- ssm_states: Optional[torch.Tensor]) -> None:
490
+ ssm_states: Optional[torch.Tensor],
491
+ pad_slot_id: int) -> None:
492
return None
493
494
@@ -800,33 +803,37 @@ def causal_conv1d_fwd(x: torch.Tensor, weight: torch.Tensor,
803
query_start_loc: Optional[torch.Tensor],
804
cache_indices: Optional[torch.Tensor],
805
has_initial_state: Optional[torch.Tensor],
803
- silu_activation: bool) -> torch.Tensor:
804
- return torch.ops._C.causal_conv1d_fwd(x, weight, bias_, conv_states,
805
- query_start_loc, cache_indices,
806
- has_initial_state, silu_activation)
807
-
808
-
809
-def causal_conv1d_update(
810
- x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
811
- bias_: Optional[torch.Tensor], silu_activation: bool,
812
- cache_seqlens: Optional[torch.Tensor],
813
- conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
814
- return torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
815
- silu_activation, cache_seqlens,
816
- conv_state_indices)
817
-
818
-
819
-def selective_scan_fwd(
820
- u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor, B: torch.Tensor,
821
- C: torch.Tensor, D_: Optional[torch.Tensor],
822
- z_: Optional[torch.Tensor], delta_bias_: Optional[torch.Tensor],
823
- delta_softplus: bool, query_start_loc: Optional[torch.Tensor],
824
- cache_indices: Optional[torch.Tensor],
825
- has_initial_state: Optional[torch.Tensor], ssm_states: torch.Tensor):
806
+ silu_activation: bool, pad_slot_id: int):
807
+ torch.ops._C.causal_conv1d_fwd(x, weight, bias_, conv_states,
808
+ query_start_loc, cache_indices,
809
+ has_initial_state, silu_activation,
810
+ pad_slot_id)
811
+
812
+
813
+def causal_conv1d_update(x: torch.Tensor, conv_state: torch.Tensor,
814
+ weight: torch.Tensor, bias_: Optional[torch.Tensor],
815
+ silu_activation: bool,
816
+ cache_seqlens: Optional[torch.Tensor],
817
+ conv_state_indices: Optional[torch.Tensor],
818
+ pad_slot_id: int):
819
+ torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
820
+ silu_activation, cache_seqlens,
821
+ conv_state_indices, pad_slot_id)
822
+
823
+
824
+def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
825
+ B: torch.Tensor, C: torch.Tensor,
826
+ D_: Optional[torch.Tensor], z_: Optional[torch.Tensor],
827
+ delta_bias_: Optional[torch.Tensor],
828
+ delta_softplus: bool,
829
+ query_start_loc: Optional[torch.Tensor],
830
+ cache_indices: Optional[torch.Tensor],
831
+ has_initial_state: Optional[torch.Tensor],
832
+ ssm_states: torch.Tensor, pad_slot_id: int):
833
torch.ops._C.selective_scan_fwd(u, delta, A, B, C, D_, z_, delta_bias_,
834
delta_softplus, query_start_loc,
835
cache_indices, has_initial_state,
829
- ssm_states)
836
+ ssm_states, pad_slot_id)
837
838
839
# moe
vllm/model_executor/layers/mamba/ops/causal_conv1d.py
+33
-20
@@ -6,18 +6,18 @@ from typing import Optional
6
import torch
7
8
from vllm import _custom_ops as ops
9
+from vllm.attention.backends.utils import PAD_SLOT_ID
10
11
11
-def causal_conv1d_fn(
12
- x: torch.Tensor,
13
- weight: torch.Tensor,
14
- bias: Optional[torch.Tensor] = None,
15
- query_start_loc: Optional[torch.Tensor] = None,
16
- cache_indices: Optional[torch.Tensor] = None,
17
- has_initial_state: Optional[torch.Tensor] = None,
18
- conv_states: Optional[torch.Tensor] = None,
19
- activation: Optional[str] = "silu",
20
-):
12
+def causal_conv1d_fn(x: torch.Tensor,
13
+ weight: torch.Tensor,
14
+ bias: Optional[torch.Tensor] = None,
15
+ query_start_loc: Optional[torch.Tensor] = None,
16
+ cache_indices: Optional[torch.Tensor] = None,
17
+ has_initial_state: Optional[torch.Tensor] = None,
18
+ conv_states: Optional[torch.Tensor] = None,
19
+ activation: Optional[str] = "silu",
20
+ pad_slot_id: int = PAD_SLOT_ID):
21
"""
22
x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen
23
sequences are concatenated from left to right for varlen
@@ -37,6 +37,13 @@ def causal_conv1d_fn(
37
conv_states: (...,dim,width - 1) itype
38
updated inplace if provided
39
activation: either None or "silu" or "swish"
40
+ pad_slot_id: int
41
+ if cache_indices is passed, lets the kernel identify padded
42
+ entries that will not be processed,
43
+ for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
44
+ in this case, the kernel will not process entries at
45
+ indices 0 and 3
46
+
47
48
out: (batch, dim, seqlen)
49
"""
@@ -46,10 +53,10 @@ def causal_conv1d_fn(
53
x = x.contiguous()
54
bias = bias.contiguous() if bias is not None else None
55
49
- out = ops.causal_conv1d_fwd(x, weight, bias, conv_states, query_start_loc,
50
- cache_indices, has_initial_state, activation
51
- in ["silu", "swish"])
52
- return out
56
+ ops.causal_conv1d_fwd(x, weight, bias, conv_states, query_start_loc,
57
+ cache_indices, has_initial_state, activation
58
+ in ["silu", "swish"], pad_slot_id)
59
+ return x
60
61
62
def causal_conv1d_update(x: torch.Tensor,
@@ -58,7 +65,8 @@ def causal_conv1d_update(x: torch.Tensor,
65
bias: Optional[torch.Tensor] = None,
66
activation: Optional[str] = None,
67
cache_seqlens: Optional[torch.Tensor] = None,
61
- conv_state_indices: Optional[torch.Tensor] = None):
68
+ conv_state_indices: Optional[torch.Tensor] = None,
69
+ pad_slot_id: int = PAD_SLOT_ID):
70
"""
71
x: (batch, dim) or (batch, dim, seqlen)
72
conv_state: (batch, dim, state_len), where state_len >= width - 1
@@ -73,7 +81,12 @@ def causal_conv1d_update(x: torch.Tensor,
81
If not None, the conv_state is a larger tensor along the batch dim,
82
and we are selecting the batch coords specified by conv_state_indices.
83
Useful for a continuous batching scenario.
76
-
84
+ pad_slot_id: int
85
+ if cache_indices is passed, lets the kernel identify padded
86
+ entries that will not be processed,
87
+ for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id]
88
+ in this case, the kernel will not process entries at
89
+ indices 0 and 3
90
out: (batch, dim) or (batch, dim, seqlen)
91
"""
92
if activation not in [None, "silu", "swish"]:
@@ -82,8 +95,8 @@ def causal_conv1d_update(x: torch.Tensor,
95
unsqueeze = x.dim() == 2
96
if unsqueeze:
97
x = x.unsqueeze(-1)
85
- out = ops.causal_conv1d_update(x, conv_state, weight, bias, activation_val,
86
- cache_seqlens, conv_state_indices)
98
+ ops.causal_conv1d_update(x, conv_state, weight, bias, activation_val,
99
+ cache_seqlens, conv_state_indices, pad_slot_id)
100
if unsqueeze:
88
- out = out.squeeze(-1)
89
- return out
101
+ x = x.squeeze(-1)
102
+ return x
vllm/model_executor/layers/mamba/ops/mamba_ssm.py
+43
-27
@@ -1,14 +1,13 @@
1
# Copyright (c) 2024, Tri Dao, Albert Gu.
2
# Adapted from https://github.com/state-spaces/mamba/blob/main/mamba_ssm/ops/triton/selective_state_update.py
3
4
-from typing import Tuple
5
-
4
import torch
5
import triton
6
import triton.language as tl
7
from packaging import version
8
9
from vllm import _custom_ops as ops
10
+from vllm.attention.backends.utils import PAD_SLOT_ID
11
12
TRITON3 = version.parse(triton.__version__) >= version.parse("3.0.0")
13
@@ -50,6 +49,7 @@ def _selective_scan_update_kernel(
49
z_ptr,
50
out_ptr,
51
state_batch_indices_ptr,
52
+ pad_slot_id,
53
# Matrix dimensions
54
batch,
55
nheads,
@@ -143,10 +143,11 @@ def _selective_scan_update_kernel(
143
if HAS_Z:
144
z_ptrs = z_ptr + offs_m * stride_z_dim
145
out_ptrs = out_ptr + offs_m * stride_out_dim
146
+ mask = (offs_m[:, None] < dim) & (offs_n[None, :] < dstate)
147
+ if HAS_STATE_BATCH_INDICES:
148
+ mask &= (state_batch_idx != pad_slot_id)
149
+ state = tl.load(state_ptrs, mask=mask, other=0.0)
150
147
- state = tl.load(state_ptrs,
148
- mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate),
149
- other=0.0)
151
x = tl.load(x_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
152
if not TIE_HDIM:
153
dt = tl.load(dt_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
@@ -177,9 +178,11 @@ def _selective_scan_update_kernel(
178
179
dB = B[None, :] * dt[:, None] if not TIE_HDIM else B * dt
180
state = state * dA + dB * x[:, None]
180
- tl.store(state_ptrs,
181
- state,
182
- mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate))
181
+
182
+ mask = (offs_m[:, None] < dim) & (offs_n[None, :] < dstate)
183
+ if HAS_STATE_BATCH_INDICES:
184
+ mask &= (state_batch_idx != pad_slot_id)
185
+ tl.store(state_ptrs, state, mask=mask)
186
out = tl.sum(state * C[None, :], axis=1)
187
if HAS_D:
188
out += x * D
@@ -198,7 +201,8 @@ def selective_state_update(state,
201
z=None,
202
dt_bias=None,
203
dt_softplus=False,
201
- state_batch_indices=None):
204
+ state_batch_indices=None,
205
+ pad_slot_id=PAD_SLOT_ID):
206
"""
207
Argument:
208
state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
@@ -210,6 +214,12 @@ def selective_state_update(state,
214
D: (dim,) or (nheads, dim)
215
z: (batch, dim) or (batch, nheads, dim)
216
dt_bias: (dim,) or (nheads, dim)
217
+ pad_slot_id: int
218
+ if cache_indices is passed, lets the kernel identify padded
219
+ entries that will not be processed,
220
+ for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
221
+ in this case, the kernel will not process entries at
222
+ indices 0 and 3
223
Return:
224
out: (batch, dim) or (batch, nheads, dim)
225
"""
@@ -276,6 +286,7 @@ def selective_state_update(state,
286
z,
287
out,
288
state_batch_indices,
289
+ pad_slot_id,
290
batch,
291
nheads,
292
dim,
@@ -319,22 +330,25 @@ def selective_state_update(state,
330
return out
331
332
322
-def selective_scan_fn(
323
- u,
324
- ssm_states,
325
- delta,
326
- A,
327
- B,
328
- C,
329
- D=None,
330
- z=None,
331
- delta_bias=None,
332
- delta_softplus=False,
333
- query_start_loc=None,
334
- cache_indices=None,
335
- has_initial_state=None) -> Tuple[torch.Tensor, torch.Tensor]:
333
+def selective_scan_fn(u,
334
+ ssm_states,
335
+ delta,
336
+ A,
337
+ B,
338
+ C,
339
+ D=None,
340
+ z=None,
341
+ delta_bias=None,
342
+ delta_softplus=False,
343
+ query_start_loc=None,
344
+ cache_indices=None,
345
+ has_initial_state=None,
346
+ pad_slot_id=PAD_SLOT_ID) -> torch.Tensor:
347
"""
348
u: (dim, total_length) for varlen or (batch, dim, seqlen)
349
+ applies changes in place.
350
+ ssm_states: (batch, dim, dstate) or (batch, nheads, dim, dstate)
351
+ applies changes in place.
352
delta: (dim, total_length) for varlen or (batch, dim, seqlen)
353
A: (dim, dstate)
354
B: (ngroups, dstate, total_length) for varlen or
@@ -357,12 +371,14 @@ def selective_scan_fn(
371
indicate if the ssm_state at the corresponding index should be
372
used as initial state. Not providing argument assumes
373
there's no initial state
360
-
374
+ pad_slot_id: int
375
+ if cache_indices is passed, lets the kernel identify padding entries
376
+ that will not be processed,
377
+ for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id]
378
+ in this case, the kernel will not process entries at indices 0 and 3
379
returns
380
output: (dim, total_length) for varlen or (batch, dim, seqlen)
381
supports inplace replacement
364
- last_state has shape (batch, dim, dstate).
365
- supports inplace replacement if ssm_state was provided
382
"""
383
if u.stride(-1) != 1:
384
u = u.contiguous()
@@ -387,7 +403,7 @@ def selective_scan_fn(
403
404
ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias, delta_softplus,
405
query_start_loc, cache_indices, has_initial_state,
390
- ssm_states)
406
+ ssm_states, pad_slot_id)
407
408
if z is None:
409
return delta # output written inplace to delta
vllm/model_executor/models/jamba.py
+31
-40
@@ -1,6 +1,5 @@
1
# coding=utf-8
2
"""Inference-only Jamba model."""
3
-from dataclasses import dataclass
3
from typing import Iterable, List, Optional, Tuple
4
5
import torch
@@ -29,7 +28,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
28
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
29
from vllm.model_executor.model_loader.weight_utils import (
30
composed_weight_loader, default_weight_loader, sharded_weight_loader)
32
-from vllm.model_executor.models.mamba_cache import MambaCacheManager
31
+from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
32
+ MambaCacheParams)
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.model_executor.utils import set_weight_attrs
35
from vllm.sequence import IntermediateTensors
@@ -41,13 +41,6 @@ from .interfaces import HasInnerState, SupportsLoRA
41
KVCache = Tuple[torch.Tensor, torch.Tensor]
42
43
44
-@dataclass
45
-class MambaCacheParams:
46
- is_prompt: bool = False
47
- conv_state: torch.Tensor = torch.Tensor()
48
- ssm_state: torch.Tensor = torch.Tensor()
49
-
50
-
44
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
45
class JambaMambaMixer(nn.Module):
46
"""
@@ -60,10 +53,9 @@ class JambaMambaMixer(nn.Module):
53
**selective** state spaces)
54
"""
55
63
- def __init__(self, config: JambaConfig, layer_idx):
56
+ def __init__(self, config: JambaConfig):
57
super().__init__()
58
self.config = config
66
- self.layer_idx = layer_idx
59
self.hidden_size = config.hidden_size
60
self.ssm_state_size = config.mamba_d_state
61
self.conv_kernel_size = config.mamba_d_conv
@@ -129,8 +121,8 @@ class JambaMambaMixer(nn.Module):
121
eps=config.rms_norm_eps)
122
123
def forward(self, hidden_states: torch.Tensor,
132
- attn_metadata: AttentionMetadata, conv_state: torch.Tensor,
133
- ssm_state: torch.Tensor):
124
+ attn_metadata: AttentionMetadata,
125
+ mamba_cache_params: MambaCacheParams):
126
127
# 1. Gated MLP's linear projection
128
projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
@@ -153,17 +145,18 @@ class JambaMambaMixer(nn.Module):
145
conv_weights,
146
self.conv1d.bias,
147
activation=self.activation,
156
- conv_states=conv_state,
148
+ conv_states=mamba_cache_params.conv_state,
149
has_initial_state=attn_metadata.context_lens_tensor > 0,
150
+ cache_indices=mamba_cache_params.state_indices_tensor,
151
query_start_loc=attn_metadata.query_start_loc)
152
else:
153
hidden_states = causal_conv1d_update(
154
hidden_states.transpose(0, 1),
162
- conv_state,
155
+ mamba_cache_params.conv_state,
156
conv_weights,
157
self.conv1d.bias,
158
self.activation,
166
- )
159
+ conv_state_indices=mamba_cache_params.state_indices_tensor)
160
hidden_states = hidden_states.transpose(0, 1)
161
162
# 3. State Space Model sequence transformation
@@ -188,7 +181,7 @@ class JambaMambaMixer(nn.Module):
181
and attn_metadata.context_lens_tensor is not None:
182
scan_outputs = selective_scan_fn(
183
hidden_states,
191
- ssm_state,
184
+ mamba_cache_params.ssm_state,
185
discrete_time_step,
186
self.A,
187
B.transpose(-2, -1),
@@ -197,11 +190,12 @@ class JambaMambaMixer(nn.Module):
190
gate,
191
time_proj_bias,
192
delta_softplus=True,
193
+ cache_indices=mamba_cache_params.state_indices_tensor,
194
has_initial_state=attn_metadata.context_lens_tensor > 0,
195
query_start_loc=attn_metadata.query_start_loc)
196
else:
197
scan_outputs = selective_state_update(
204
- ssm_state,
198
+ mamba_cache_params.ssm_state,
199
hidden_states.transpose(0, 1),
200
discrete_time_step.transpose(0, 1),
201
self.A,
@@ -211,7 +205,7 @@ class JambaMambaMixer(nn.Module):
205
gate.transpose(0, 1),
206
time_proj_bias,
207
dt_softplus=True,
214
- )
208
+ state_batch_indices=mamba_cache_params.state_indices_tensor)
209
scan_outputs = scan_outputs.transpose(0, 1)
210
211
# 4. Final linear projection
@@ -292,7 +286,7 @@ class JambaMambaDecoderLayer(nn.Module):
286
super().__init__()
287
self.layer_idx = layer_idx
288
self.config = config
295
- self.mamba = JambaMambaMixer(config, layer_idx)
289
+ self.mamba = JambaMambaMixer(config)
290
291
num_experts = config.layers_num_experts[layer_idx]
292
ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
@@ -307,8 +301,7 @@ class JambaMambaDecoderLayer(nn.Module):
301
hidden_states: torch.Tensor,
302
attn_metadata: AttentionMetadata,
303
residual: Optional[torch.Tensor],
310
- conv_state: torch.Tensor,
311
- ssm_state: torch.Tensor,
304
+ mamba_cache_params: MambaCacheParams,
305
**kwargs,
306
):
307
if residual is None:
@@ -318,8 +311,8 @@ class JambaMambaDecoderLayer(nn.Module):
311
hidden_states, residual = self.input_layernorm(
312
hidden_states, residual)
313
321
- hidden_states = self.mamba(hidden_states, attn_metadata, conv_state,
322
- ssm_state)
314
+ hidden_states = self.mamba(hidden_states, attn_metadata,
315
+ mamba_cache_params)
316
# Fully Connected
317
hidden_states, residual = self.pre_ff_layernorm(
318
hidden_states, residual)
@@ -476,17 +469,14 @@ class JambaModel(nn.Module):
469
positions: torch.Tensor,
470
kv_caches: List[torch.Tensor],
471
attn_metadata: AttentionMetadata,
479
- conv_state: torch.Tensor,
480
- ssm_state: torch.Tensor,
472
+ mamba_cache_params: MambaCacheParams,
473
) -> torch.Tensor:
474
hidden_states = self.embed_tokens(input_ids)
475
residual = None
484
-
476
for i in range(len(self.layers)):
477
layer = self.layers[i]
478
kv_cache = None
488
- current_ssm_state = None
489
- current_conv_state = None
479
+ layer_mamba_cache_params = None
480
if isinstance(layer, JambaAttentionDecoderLayer):
481
kv_cache = kv_caches[(i - self.config.attn_layer_offset) //
482
self.config.attn_layer_period]
@@ -494,8 +484,8 @@ class JambaModel(nn.Module):
484
current_state_layer = i - (1 +
485
(i - self.config.attn_layer_offset)
486
// self.config.attn_layer_period)
497
- current_ssm_state = ssm_state[current_state_layer]
498
- current_conv_state = conv_state[current_state_layer]
487
+ layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
488
+ current_state_layer)
489
490
hidden_states, residual = layer(
491
positions=positions,
@@ -503,9 +493,7 @@ class JambaModel(nn.Module):
493
kv_cache=kv_cache,
494
attn_metadata=attn_metadata,
495
residual=residual,
506
- conv_state=current_conv_state,
507
- ssm_state=current_ssm_state,
508
- )
496
+ mamba_cache_params=layer_mamba_cache_params)
497
hidden_states, _ = self.final_layernorm(hidden_states, residual)
498
return hidden_states
499
@@ -588,13 +576,16 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
576
self.mamba_cache = MambaCacheManager(
577
self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
578
*self._get_mamba_cache_shape())
591
-
592
- mamba_cache_tensors = self.mamba_cache.current_run_tensors(
593
- input_ids, attn_metadata, **kwargs)
594
-
579
+ (
580
+ mamba_cache_tensors,
581
+ state_indices_tensor,
582
+ ) = self.mamba_cache.current_run_tensors(input_ids, attn_metadata,
583
+ **kwargs)
584
+ mamba_cache_params = MambaCacheParams(mamba_cache_tensors[0],
585
+ mamba_cache_tensors[1],
586
+ state_indices_tensor)
587
hidden_states = self.model(input_ids, positions, kv_caches,
596
- attn_metadata, mamba_cache_tensors[0],
597
- mamba_cache_tensors[1])
588
+ attn_metadata, mamba_cache_params)
589
return hidden_states
590
591
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
vllm/model_executor/models/mamba.py
+28
-25
@@ -27,7 +27,8 @@ from vllm.model_executor.model_loader.weight_utils import (
27
composed_weight_loader, default_weight_loader, sharded_weight_loader)
28
from vllm.model_executor.models.interfaces import (HasInnerState,
29
IsAttentionFree)
30
-from vllm.model_executor.models.mamba_cache import MambaCacheManager
30
+from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
31
+ MambaCacheParams)
32
from vllm.model_executor.sampling_metadata import SamplingMetadata
33
from vllm.model_executor.utils import set_weight_attrs
34
from vllm.sequence import IntermediateTensors
@@ -110,8 +111,8 @@ class MambaMixer(nn.Module):
111
self.activation = config.hidden_act
112
113
def forward(self, hidden_states: torch.Tensor,
113
- attn_metadata: AttentionMetadata, conv_state: torch.Tensor,
114
- ssm_state: torch.Tensor):
114
+ attn_metadata: AttentionMetadata,
115
+ mamba_cache_params: MambaCacheParams):
116
117
# 1. Gated MLP's linear projection
118
projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
@@ -134,17 +135,18 @@ class MambaMixer(nn.Module):
135
conv_weights,
136
self.conv1d.bias,
137
activation=self.activation,
137
- conv_states=conv_state,
138
+ conv_states=mamba_cache_params.conv_state,
139
has_initial_state=attn_metadata.context_lens_tensor > 0,
140
+ cache_indices=mamba_cache_params.state_indices_tensor,
141
query_start_loc=attn_metadata.query_start_loc)
142
else:
143
hidden_states = causal_conv1d_update(
144
hidden_states.transpose(0, 1),
143
- conv_state,
145
+ mamba_cache_params.conv_state,
146
conv_weights,
147
self.conv1d.bias,
148
self.activation,
147
- )
149
+ conv_state_indices=mamba_cache_params.state_indices_tensor)
150
hidden_states = hidden_states.transpose(0, 1)
151
152
# 3. State Space Model sequence transformation
@@ -168,7 +170,7 @@ class MambaMixer(nn.Module):
170
and attn_metadata.context_lens_tensor is not None:
171
scan_outputs = selective_scan_fn(
172
hidden_states,
171
- ssm_state,
173
+ mamba_cache_params.ssm_state,
174
discrete_time_step,
175
self.A,
176
B.transpose(-2, -1),
@@ -177,11 +179,12 @@ class MambaMixer(nn.Module):
179
gate,
180
time_proj_bias,
181
delta_softplus=True,
182
+ cache_indices=mamba_cache_params.state_indices_tensor,
183
has_initial_state=attn_metadata.context_lens_tensor > 0,
184
query_start_loc=attn_metadata.query_start_loc)
185
else:
186
scan_outputs = selective_state_update(
184
- ssm_state,
187
+ mamba_cache_params.ssm_state,
188
hidden_states.transpose(0, 1),
189
discrete_time_step.transpose(0, 1),
190
self.A,
@@ -191,7 +194,7 @@ class MambaMixer(nn.Module):
194
gate.transpose(0, 1),
195
time_proj_bias,
196
dt_softplus=True,
194
- )
197
+ state_batch_indices=mamba_cache_params.state_indices_tensor)
198
scan_outputs = scan_outputs.transpose(0, 1)
199
200
# 4. Final linear projection
@@ -221,8 +224,7 @@ class MambaDecoderLayer(nn.Module):
224
hidden_states: torch.Tensor,
225
attn_metadata: AttentionMetadata,
226
residual: Optional[torch.Tensor],
224
- conv_state: torch.Tensor,
225
- ssm_state: torch.Tensor,
227
+ mamba_cache_params: MambaCacheParams,
228
**kwargs,
229
):
230
if residual is None:
@@ -231,8 +233,8 @@ class MambaDecoderLayer(nn.Module):
233
else:
234
hidden_states, residual = self.norm(hidden_states, residual)
235
234
- hidden_states = self.mixer(hidden_states, attn_metadata, conv_state,
235
- ssm_state)
236
+ hidden_states = self.mixer(hidden_states, attn_metadata,
237
+ mamba_cache_params)
238
return hidden_states, residual
239
240
@@ -275,25 +277,20 @@ class MambaModel(nn.Module):
277
input_ids: torch.Tensor,
278
positions: torch.Tensor,
279
attn_metadata: AttentionMetadata,
278
- conv_state: torch.Tensor,
279
- ssm_state: torch.Tensor,
280
+ mamba_cache_params: MambaCacheParams,
281
) -> torch.Tensor:
282
+
283
hidden_states = self.embeddings(input_ids)
284
residual = None
285
286
for i in range(len(self.layers)):
287
layer = self.layers[i]
286
- current_ssm_state = ssm_state[i]
287
- current_conv_state = conv_state[i]
288
-
288
hidden_states, residual = layer(
289
positions=positions,
290
hidden_states=hidden_states,
291
attn_metadata=attn_metadata,
292
residual=residual,
294
- conv_state=current_conv_state,
295
- ssm_state=current_ssm_state,
296
- )
293
+ mamba_cache_params=mamba_cache_params.at_layer_idx(i))
294
hidden_states, _ = self.norm_f(hidden_states, residual)
295
296
return hidden_states
@@ -347,12 +344,18 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
344
self.lm_head.weight.dtype, self.config.num_hidden_layers,
345
max_batch_size, *self._get_mamba_cache_shape())
346
350
- mamba_cache_tensors = self.mamba_cache.current_run_tensors(
351
- input_ids, attn_metadata, **kwargs)
347
+ (
348
+ mamba_cache_tensors,
349
+ state_indices_tensor,
350
+ ) = self.mamba_cache.current_run_tensors(input_ids, attn_metadata,
351
+ **kwargs)
352
+
353
+ mamba_cache_params = MambaCacheParams(mamba_cache_tensors[0],
354
+ mamba_cache_tensors[1],
355
+ state_indices_tensor)
356
357
hidden_states = self.backbone(input_ids, positions, attn_metadata,
354
- mamba_cache_tensors[0],
355
- mamba_cache_tensors[1])
358
+ mamba_cache_params)
359
360
return hidden_states
361
vllm/model_executor/models/mamba_cache.py
+61
-125
@@ -1,8 +1,22 @@
1
-from typing import Dict, List, Optional
1
+from dataclasses import dataclass
2
+from typing import Dict, List
3
4
import torch
5
6
from vllm.attention.backends.abstract import AttentionMetadata
7
+from vllm.attention.backends.utils import PAD_SLOT_ID
8
+
9
+
10
+@dataclass
11
+class MambaCacheParams:
12
+ conv_state: torch.Tensor = torch.Tensor()
13
+ ssm_state: torch.Tensor = torch.Tensor()
14
+ state_indices_tensor: torch.Tensor = torch.Tensor()
15
+
16
+ def at_layer_idx(self, layer_idx):
17
+ return MambaCacheParams(self.conv_state[layer_idx],
18
+ self.ssm_state[layer_idx],
19
+ self.state_indices_tensor)
20
21
22
class MambaCacheManager:
@@ -24,6 +38,7 @@ class MambaCacheManager:
38
# Maps between the request id and a dict that maps between the seq_id
39
# and its index inside the self.mamba_cache
40
self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
41
+ self.free_cache_indices = list(range(max_batch_size))
42
43
def current_run_tensors(self, input_ids: torch.Tensor,
44
attn_metadata: AttentionMetadata, **kwargs):
@@ -36,30 +51,43 @@ class MambaCacheManager:
51
finished_requests_ids = kwargs["finished_requests_ids"]
52
53
self._release_finished_requests(finished_requests_ids)
39
- mamba_cache_tensors = self._prepare_current_run_mamba_cache(
54
+ state_indices = self._prepare_current_run_mamba_cache(
55
request_ids_to_seq_ids, finished_requests_ids)
56
57
+ state_indices_tensor = torch.as_tensor(state_indices,
58
+ dtype=torch.int32,
59
+ device="cuda")
60
+ mamba_cache_tensors = self.mamba_cache
61
+
62
else:
63
# CUDA graph capturing runs
44
- mamba_cache_tensors = kwargs["seqlen_agnostic_capture_inputs"]
64
+ (mamba_cache_tensors,
65
+ state_indices_tensor) = kwargs["seqlen_agnostic_capture_inputs"]
66
46
- return mamba_cache_tensors
67
+ return (mamba_cache_tensors, state_indices_tensor)
68
69
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
70
"""
50
- Copy the relevant Mamba cache into the CUDA graph input buffer
51
- that was provided during the capture runs
52
- (JambaForCausalLM.mamba_gc_cache_buffer).
71
+ Copy the relevant state_indices into the CUDA graph input buffer
72
"""
73
assert all(
74
key in kwargs
75
for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
76
finished_requests_ids = kwargs["finished_requests_ids"]
77
request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
78
+ assert "seqlen_agnostic_capture_inputs" in input_buffers
79
+ _, input_state_indices_buffer = input_buffers[
80
+ "seqlen_agnostic_capture_inputs"]
81
82
self._release_finished_requests(finished_requests_ids)
61
- self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
62
- finished_requests_ids)
83
+ state_indices = self._prepare_current_run_mamba_cache(
84
+ request_ids_to_seq_ids, finished_requests_ids)
85
+ cuda_graph_pad_len = input_state_indices_buffer.shape[0] - len(
86
+ state_indices)
87
+ state_indices.extend([PAD_SLOT_ID] * cuda_graph_pad_len)
88
+
89
+ input_state_indices_buffer.copy_(
90
+ torch.as_tensor(state_indices, dtype=torch.int32, device="cuda"))
91
92
def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
93
"""
@@ -67,13 +95,10 @@ class MambaCacheManager:
95
The buffer is used to maintain the Mamba Cache during the CUDA graph
96
replay runs.
97
"""
70
- return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache)
71
-
72
- def _swap_mamba_cache(self, from_index: int, to_index: int):
73
- assert len(self.mamba_cache) > 0
74
- for cache_t in self.mamba_cache:
75
- cache_t[:, [to_index,from_index]] = \
76
- cache_t[:, [from_index,to_index]]
98
+ state_indices_tensor = torch.as_tensor([PAD_SLOT_ID] * batch_size,
99
+ dtype=torch.int32,
100
+ device="cuda")
101
+ return (self.mamba_cache, state_indices_tensor)
102
103
def _copy_mamba_cache(self, from_index: int, to_index: int):
104
assert len(self.mamba_cache) > 0
@@ -81,142 +106,53 @@ class MambaCacheManager:
106
cache_t[:, to_index].copy_(cache_t[:, from_index],
107
non_blocking=True)
108
84
- def _move_out_if_already_occupied(self, index: int,
85
- all_occupied_indices: List[int]):
86
- if index in all_occupied_indices:
87
- first_free_index = self._first_free_index_in_mamba_cache()
88
- # In case occupied, move the occupied to a new empty block
89
- self._move_cache_index_and_mappings(from_index=index,
90
- to_index=first_free_index)
91
-
92
- def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str,
93
- seq_id: int,
94
- destination_index: int):
109
+ def _assign_seq_id_to_cache_index(self, cur_rid: str, seq_id: int,
110
+ finished_requests_ids) -> int:
111
"""
112
Assign (req_id,seq_id) pair to a `destination_index` index, if
113
already occupied, move the occupying index to a free index.
114
"""
99
- all_occupied_indices = self._get_all_occupied_indices()
100
- if cur_rid not in self.mamba_cache_indices_mapping:
101
- self._move_out_if_already_occupied(
102
- index=destination_index,
103
- all_occupied_indices=all_occupied_indices)
115
+ if cur_rid in finished_requests_ids:
116
+ # set as pad, do not allocate destination index
117
+ return PAD_SLOT_ID
118
+ elif cur_rid not in self.mamba_cache_indices_mapping:
119
+ destination_index = self.free_cache_indices.pop()
120
self.mamba_cache_indices_mapping[cur_rid] = {
121
seq_id: destination_index
122
}
123
+ return destination_index
124
elif seq_id not in (seq_ids2indices :=
125
self.mamba_cache_indices_mapping[cur_rid]):
126
# parallel sampling , where n > 1, assume prefill have
110
- # already happened now we only need to copy the already
127
+ # already happened, so we copy the
128
# existing cache into the siblings seq_ids caches
112
- self._move_out_if_already_occupied(
113
- index=destination_index,
114
- all_occupied_indices=all_occupied_indices)
115
- index_exists = list(seq_ids2indices.values())[0]
129
+ index_exists = next(iter(seq_ids2indices.values()))
130
# case of decoding n>1, copy prefill cache to decoding indices
131
+ destination_index = self.free_cache_indices.pop()
132
self._copy_mamba_cache(from_index=index_exists,
133
to_index=destination_index)
134
self.mamba_cache_indices_mapping[cur_rid][
135
seq_id] = destination_index
136
+ return destination_index
137
else:
138
# already exists
123
- cache_index_already_exists = self.mamba_cache_indices_mapping[
124
- cur_rid][seq_id]
125
- if cache_index_already_exists != destination_index:
126
- # In case the seq id already exists but not in
127
- # the right destination, swap it with what's occupying it
128
- self._swap_pair_indices_and_mappings(
129
- from_index=cache_index_already_exists,
130
- to_index=destination_index)
139
+ return self.mamba_cache_indices_mapping[cur_rid][seq_id]
140
141
def _prepare_current_run_mamba_cache(
142
self, request_ids_to_seq_ids: Dict[str, list[int]],
134
- finished_requests_ids: List[str]):
135
- running_indices = []
136
- request_ids_to_seq_ids_flatten = [
137
- (req_id, seq_id)
143
+ finished_requests_ids: List[str]) -> List[int]:
144
+ return [
145
+ self._assign_seq_id_to_cache_index(req_id, seq_id,
146
+ finished_requests_ids)
147
for req_id, seq_ids in request_ids_to_seq_ids.items()
148
for seq_id in seq_ids
149
]
141
- batch_size = len(request_ids_to_seq_ids_flatten)
142
- for dest_index, (request_id,
143
- seq_id) in enumerate(request_ids_to_seq_ids_flatten):
144
- if request_id in finished_requests_ids:
145
- # Do not allocate cache index for requests that run
146
- # and finish right after
147
- continue
148
- self._assign_seq_id_to_mamba_cache_in_specific_dest(
149
- request_id, seq_id, dest_index)
150
- running_indices.append(dest_index)
151
-
152
- self._clean_up_first_bs_blocks(batch_size, running_indices)
153
- conv_state = self.mamba_cache[0][:, :batch_size]
154
- temporal_state = self.mamba_cache[1][:, :batch_size]
155
-
156
- return (conv_state, temporal_state)
157
-
158
- def _get_all_occupied_indices(self):
159
- return [
160
- cache_idx
161
- for seq_ids2indices in self.mamba_cache_indices_mapping.values()
162
- for cache_idx in seq_ids2indices.values()
163
- ]
164
-
165
- def _clean_up_first_bs_blocks(self, batch_size: int,
166
- indices_for_current_run: List[int]):
167
- # move out all of the occupied but currently not running blocks
168
- # outside of the first n blocks
169
- destination_indices = range(batch_size)
170
- max_possible_batch_size = self.mamba_cache[0].shape[1]
171
- for destination_index in destination_indices:
172
- if destination_index in self._get_all_occupied_indices() and \
173
- destination_index not in indices_for_current_run:
174
- # move not running indices outside of the batch
175
- all_other_indices = list(
176
- range(batch_size, max_possible_batch_size))
177
- first_avail_index = self._first_free_index_in_mamba_cache(
178
- all_other_indices)
179
- self._swap_indices(from_index=destination_index,
180
- to_index=first_avail_index)
181
-
182
- def _move_cache_index_and_mappings(self, from_index: int, to_index: int):
183
- self._copy_mamba_cache(from_index=from_index, to_index=to_index)
184
- self._update_mapping_index(from_index=from_index, to_index=to_index)
185
-
186
- def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int):
187
- self._swap_mamba_cache(from_index=from_index, to_index=to_index)
188
- self._swap_mapping_index(from_index=from_index, to_index=to_index)
189
-
190
- def _swap_mapping_index(self, from_index: int, to_index: int):
191
- for seq_ids2index in self.mamba_cache_indices_mapping.values():
192
- for seq_id, index in seq_ids2index.items():
193
- if from_index == index:
194
- seq_ids2index.update({seq_id: to_index})
195
- elif to_index == index:
196
- seq_ids2index.update({seq_id: from_index})
197
-
198
- def _update_mapping_index(self, from_index: int, to_index: int):
199
- for seq_ids2index in self.mamba_cache_indices_mapping.values():
200
- for seq_id, index in seq_ids2index.items():
201
- if from_index == index:
202
- seq_ids2index.update({seq_id: to_index})
203
- return
150
151
def _release_finished_requests(self,
152
finished_seq_groups_req_ids: List[str]):
153
for req_id in finished_seq_groups_req_ids:
154
if req_id in self.mamba_cache_indices_mapping:
155
+ for seq_id in self.mamba_cache_indices_mapping[req_id]:
156
+ self.free_cache_indices.append(
157
+ self.mamba_cache_indices_mapping[req_id][seq_id])
158
self.mamba_cache_indices_mapping.pop(req_id)
210
-
211
- def _first_free_index_in_mamba_cache(
212
- self, indices_range: Optional[List[int]] = None) -> int:
213
- assert self.mamba_cache is not None
214
- if indices_range is None:
215
- max_possible_batch_size = self.mamba_cache[0].shape[1]
216
- indices_range = list(range(max_possible_batch_size))
217
- all_occupied_indices = self._get_all_occupied_indices()
218
- for i in indices_range:
219
- if i not in all_occupied_indices:
220
- return i
221
- raise Exception("Couldn't find a free spot in the mamba cache! This"
222
- "should never happen")