@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fb60ae9b91
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Kernel][Model] Improve continuous batching for Jamba and Mamba (#9189)

Mor Zusman committed Oct 17, 2024 at 00:12 UTC fb60ae9b91a4b3e1aed4a6e826895fe3c5a13c10
15 files changed +511 -439
csrc/mamba/causal_conv1d/causal_conv1d.cu
+25 -12
@@ -55,6 +55,7 @@ void set_conv_params_fwd(ConvParamsBase &params,
55 const at::Tensor out,
56 const c10::optional<at::Tensor>& bias,
57 bool silu_activation,
58 + int64_t pad_slot_id,
59 const c10::optional<at::Tensor>& query_start_loc = std::nullopt,
60 const c10::optional<at::Tensor>& cache_indices = std::nullopt,
61 const c10::optional<at::Tensor>& has_initial_state = std::nullopt) {
@@ -66,6 +67,7 @@ void set_conv_params_fwd(ConvParamsBase &params,
67 params.dim = dim;
68 params.seqlen = seqlen;
69 params.width = width;
70 + params.pad_slot_id = pad_slot_id;
71
72 params.silu_activation = silu_activation;
73
@@ -90,14 +92,16 @@ void set_conv_params_fwd(ConvParamsBase &params,
92 }
93
94
93 -at::Tensor
94 -causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
95 +void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
96 const c10::optional<at::Tensor> &bias_,
97 const c10::optional<at::Tensor> &conv_states,
98 const c10::optional<at::Tensor> &query_start_loc,
99 const c10::optional<at::Tensor> &cache_indices,
100 const c10::optional<at::Tensor> &has_initial_state,
100 - bool silu_activation) {
101 + bool silu_activation,
102 + // used to identify padding entries if cache_indices provided
103 + // in case of padding, the kernel will return early
104 + int64_t pad_slot_id) {
105 auto input_type = x.scalar_type();
106 auto weight_type = weight.scalar_type();
107 TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -153,12 +157,13 @@ causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
157 CHECK_SHAPE(cache_indices_, batch_size);
158 }
159
156 - at::Tensor out = torch::empty_like(x);
160 + at::Tensor out = x;
161
162 ConvParamsBase params;
163 set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
164 bias_,
165 silu_activation,
166 + pad_slot_id,
167 query_start_loc,
168 cache_indices,
169 has_initial_state
@@ -183,18 +188,19 @@ causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
188 DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_fwd", [&] {
189 causal_conv1d_fwd_cuda<input_t, weight_t>(params, stream);
190 });
186 - return out;
191 }
192
193
190 -at::Tensor
191 -causal_conv1d_update(const at::Tensor &x,
194 +void causal_conv1d_update(const at::Tensor &x,
195 const at::Tensor &conv_state,
196 const at::Tensor &weight,
197 const c10::optional<at::Tensor> &bias_,
198 bool silu_activation,
199 const c10::optional<at::Tensor> &cache_seqlens_,
197 - const c10::optional<at::Tensor> &conv_state_indices_) {
200 + const c10::optional<at::Tensor> &conv_state_indices_,
201 + // used to identify padding entries if cache_indices provided
202 + // in case of padding, the kernel will return early
203 + int64_t pad_slot_id) {
204 auto input_type = x.scalar_type();
205 auto weight_type = weight.scalar_type();
206 TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -227,12 +233,13 @@ causal_conv1d_update(const at::Tensor &x,
233 CHECK_SHAPE(bias, dim);
234 }
235
230 - at::Tensor out = torch::empty_like(x);
236 + at::Tensor out = x;
237
238 ConvParamsBase params;
239 set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
240 bias_,
235 - silu_activation);
241 + silu_activation,
242 + pad_slot_id);
243 params.conv_state_ptr = conv_state.data_ptr();
244 params.conv_state_len = conv_state_len;
245 // All stride are in elements, not bytes.
@@ -274,7 +281,6 @@ causal_conv1d_update(const at::Tensor &x,
281 DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_update", [&] {
282 causal_conv1d_update_cuda<input_t, weight_t>(params, stream);
283 });
277 - return out;
284 }
285
286 template<int kNThreads_, int kWidth_, bool kIsVecLoad_, typename input_t_, typename weight_t_>
@@ -340,7 +346,10 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
346 int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr
347 : reinterpret_cast<int *>(params.cache_indices_ptr);
348 int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id];
343 -
349 + // cache_index == params.pad_slot_id is defined as padding, so we exit early
350 + if (cache_index == params.pad_slot_id){
351 + return;
352 + }
353 input_t *conv_states = params.conv_states_ptr == nullptr ? nullptr
354 : reinterpret_cast<input_t *>(params.conv_states_ptr) + cache_index * params.conv_states_batch_stride + channel_id * params.conv_states_c_stride;
355
@@ -528,6 +537,10 @@ void causal_conv1d_update_kernel(ConvParamsBase params) {
537 const int conv_state_batch_coord = params.conv_state_indices_ptr == nullptr
538 ? batch_id
539 : params.conv_state_indices_ptr[batch_id];
540 + // conv_state_batch_coord == params.pad_slot_id is defined as padding so we exit early
541 + if (conv_state_batch_coord == params.pad_slot_id){
542 + return;
543 + }
544 input_t *conv_state = reinterpret_cast<input_t *>(params.conv_state_ptr)
545 + conv_state_batch_coord * params.conv_state_batch_stride
546 + channel_id * params.conv_state_c_stride;
csrc/mamba/causal_conv1d/causal_conv1d.h
+1
@@ -13,6 +13,7 @@ struct ConvParamsBase {
13 using index_t = uint32_t;
14
15 int batch, dim, seqlen, width;
16 + int64_t pad_slot_id;
17 bool silu_activation;
18
19 index_t x_batch_stride;
csrc/mamba/mamba_ssm/selective_scan.h
+1
@@ -21,6 +21,7 @@ struct SSMParamsBase {
21 int dim_ngroups_ratio;
22 bool is_variable_B;
23 bool is_variable_C;
24 + int64_t pad_slot_id;
25
26 bool delta_softplus;
27
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
+14 -10
@@ -115,6 +115,10 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
115 const int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr
116 : reinterpret_cast<int *>(params.cache_indices_ptr);
117 const int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id];
118 + // cache_index == params.pad_slot_id is defined as padding, so we exit early
119 + if (cache_index == params.pad_slot_id){
120 + return;
121 + }
122 input_t *u = reinterpret_cast<input_t *>(params.u_ptr) + sequence_start_index * params.u_batch_stride
123 + dim_id * kNRows * params.u_d_stride;
124 input_t *delta = reinterpret_cast<input_t *>(params.delta_ptr) + sequence_start_index * params.delta_batch_stride
@@ -387,7 +391,6 @@ void set_ssm_params_fwd(SSMParamsBase &params,
391 const size_t seqlen,
392 const size_t dstate,
393 const size_t n_groups,
390 - const size_t n_chunks,
394 const bool is_variable_B,
395 const bool is_variable_C,
396 // device pointers
@@ -407,7 +410,8 @@ void set_ssm_params_fwd(SSMParamsBase &params,
410 const c10::optional<at::Tensor>& query_start_loc,
411 const c10::optional<at::Tensor>& cache_indices,
412 const c10::optional<at::Tensor>& has_initial_state,
410 - bool varlen) {
413 + bool varlen,
414 + int64_t pad_slot_id) {
415
416 // Reset the parameters
417 memset(&params, 0, sizeof(params));
@@ -417,8 +421,8 @@ void set_ssm_params_fwd(SSMParamsBase &params,
421 params.seqlen = seqlen;
422 params.dstate = dstate;
423 params.n_groups = n_groups;
420 - params.n_chunks = n_chunks;
424 params.dim_ngroups_ratio = dim / n_groups;
425 + params.pad_slot_id = pad_slot_id;
426
427 params.delta_softplus = delta_softplus;
428
@@ -507,7 +511,10 @@ void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
511 const c10::optional<torch::Tensor> &query_start_loc,
512 const c10::optional<torch::Tensor> &cache_indices,
513 const c10::optional<torch::Tensor> &has_initial_state,
510 - const torch::Tensor &ssm_states) {
514 + const torch::Tensor &ssm_states,
515 + // used to identify padding entries if cache_indices provided
516 + // in case of padding, the kernel will return early
517 + int64_t pad_slot_id) {
518 auto input_type = u.scalar_type();
519 auto weight_type = A.scalar_type();
520 TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -618,18 +625,14 @@ void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
625
626 out_z = z;
627
621 - const int n_chunks = (seqlen + 2048 - 1) / 2048;
622 - // const int n_chunks = (seqlen + 1024 - 1) / 1024;
623 - // at::Tensor out = torch::empty_like(u);
628 // Right now u has BHL layout and delta has HBL layout, and we want out to have HBL layout
629 at::Tensor out = delta;
630 TORCH_CHECK(ssm_states.scalar_type() == input_type);
631 TORCH_CHECK(ssm_states.is_cuda());
632 TORCH_CHECK(ssm_states.stride(-1) == 1);
629 - CHECK_SHAPE(ssm_states, batch_size, dim, dstate);
633
634 SSMParamsBase params;
632 - set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, n_chunks, is_variable_B, is_variable_C,
635 + set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, is_variable_B, is_variable_C,
636 u, delta, A, B, C, out, z, out_z,
637 D_,
638 delta_bias_,
@@ -639,7 +642,8 @@ void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
642 query_start_loc,
643 cache_indices,
644 has_initial_state,
642 - varlen
645 + varlen,
646 + pad_slot_id
647 );
648
649
csrc/ops.h
+17 -15
@@ -157,21 +157,23 @@ void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
157 const c10::optional<torch::Tensor>& query_start_loc,
158 const c10::optional<torch::Tensor>& cache_indices,
159 const c10::optional<torch::Tensor>& has_initial_state,
160 - const torch::Tensor& ssm_states);
161 -
162 -at::Tensor causal_conv1d_update(
163 - const at::Tensor& x, const at::Tensor& conv_state, const at::Tensor& weight,
164 - const c10::optional<at::Tensor>& bias_, bool silu_activation,
165 - const c10::optional<at::Tensor>& cache_seqlens_,
166 - const c10::optional<at::Tensor>& conv_state_indices_);
167 -
168 -at::Tensor causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
169 - const c10::optional<at::Tensor>& bias_,
170 - const c10::optional<at::Tensor>& conv_states,
171 - const c10::optional<at::Tensor>& query_start_loc,
172 - const c10::optional<at::Tensor>& cache_indices,
173 - const c10::optional<at::Tensor>& has_initial_state,
174 - bool silu_activation);
160 + const torch::Tensor& ssm_states, int64_t pad_slot_id);
161 +
162 +void causal_conv1d_update(const at::Tensor& x, const at::Tensor& conv_state,
163 + const at::Tensor& weight,
164 + const c10::optional<at::Tensor>& bias_,
165 + bool silu_activation,
166 + const c10::optional<at::Tensor>& cache_seqlens_,
167 + const c10::optional<at::Tensor>& conv_state_indices_,
168 + int64_t pad_slot_id);
169 +
170 +void causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
171 + const c10::optional<at::Tensor>& bias_,
172 + const c10::optional<at::Tensor>& conv_states,
173 + const c10::optional<at::Tensor>& query_start_loc,
174 + const c10::optional<at::Tensor>& cache_indices,
175 + const c10::optional<at::Tensor>& has_initial_state,
176 + bool silu_activation, int64_t pad_slot_id);
177
178 #ifndef USE_ROCM
179 using fptr_t = int64_t;
csrc/torch_bindings.cpp
+6 -3
@@ -278,7 +278,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
278 "Tensor? query_start_loc,"
279 "Tensor? cache_indices,"
280 "Tensor? has_initial_state,"
281 - "Tensor! ssm_states) -> ()");
281 + "Tensor! ssm_states,"
282 + "int pad_slot_id) -> ()");
283 ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
284
285 ops.def(
@@ -288,7 +289,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
289 "Tensor? bias_,"
290 "bool silu_activation,"
291 "Tensor? cache_seqlens_,"
291 - "Tensor? conv_state_indices) -> Tensor");
292 + "Tensor? conv_state_indices,"
293 + "int pad_slot_id) -> ()");
294 ops.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update);
295
296 ops.def(
@@ -298,7 +300,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
300 "Tensor? query_start_loc,"
301 "Tensor? cache_indices,"
302 "Tensor? has_initial_state,"
301 - "bool silu_activation) -> Tensor");
303 + "bool silu_activation,"
304 + "int pad_slot_id) -> ()");
305 ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
306 #endif
307
tests/kernels/test_causal_conv1d.py
+97 -94
@@ -6,6 +6,7 @@ import torch.nn.functional as F
6
7 from tests.kernels.utils import opcheck
8 from vllm import _custom_ops as ops # noqa: F401
9 +from vllm.attention.backends.utils import PAD_SLOT_ID
10 from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
11 causal_conv1d_fn, causal_conv1d_update)
12 from vllm.utils import seed_everything
@@ -114,16 +115,15 @@ def causal_conv1d_update_ref(x,
115 @pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
116 @pytest.mark.parametrize("silu_activation", [True])
117 @pytest.mark.parametrize("has_bias", [True])
117 -def causal_conv1d_opcheck_fn(
118 - x: torch.Tensor,
119 - weight: torch.Tensor,
120 - bias: Optional[torch.Tensor] = None,
121 - cu_seq_len: Optional[torch.Tensor] = None,
122 - cache_indices: Optional[torch.Tensor] = None,
123 - has_initial_state: Optional[torch.Tensor] = None,
124 - conv_states: Optional[torch.Tensor] = None,
125 - activation: Optional[str] = "silu",
126 -):
118 +def causal_conv1d_opcheck_fn(x: torch.Tensor,
119 + weight: torch.Tensor,
120 + bias: Optional[torch.Tensor] = None,
121 + cu_seq_len: Optional[torch.Tensor] = None,
122 + cache_indices: Optional[torch.Tensor] = None,
123 + has_initial_state: Optional[torch.Tensor] = None,
124 + conv_states: Optional[torch.Tensor] = None,
125 + activation: Optional[str] = "silu",
126 + pad_slot_id: int = PAD_SLOT_ID):
127 """
128 x: (batch, dim, seqlen)
129 weight: (dim, width)
@@ -141,16 +141,9 @@ def causal_conv1d_opcheck_fn(
141 x = x.contiguous()
142 bias = bias.contiguous() if bias is not None else None
143
144 - opcheck(torch.ops._C.causal_conv1d_fwd, (
145 - x,
146 - weight,
147 - bias,
148 - conv_states,
149 - cu_seq_len,
150 - cache_indices,
151 - has_initial_state,
152 - activation in ["silu", "swish"],
153 - ))
144 + opcheck(torch.ops._C.causal_conv1d_fwd,
145 + (x, weight, bias, conv_states, cu_seq_len, cache_indices,
146 + has_initial_state, activation in ["silu", "swish"], pad_slot_id))
147
148
149 @pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
@@ -233,17 +226,11 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
226 seed_everything(0)
227 batch = 2
228 x = torch.randn(batch, dim, seqlen, device=device, dtype=itype)
229 + x_ref = x.clone()
230 conv_state = torch.randn(batch, dim, width - 1, device=device, dtype=itype)
231
238 - weight = torch.randn(dim,
239 - width,
240 - device=device,
241 - dtype=itype,
242 - requires_grad=True)
243 - if has_bias:
244 - bias = torch.randn(dim, device=device, dtype=itype, requires_grad=True)
245 - else:
246 - bias = None
232 + weight = torch.randn(dim, width, device=device, dtype=itype)
233 + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
234 conv_state_ref = conv_state.detach().clone()
235 activation = None if not silu_activation else "silu"
236 out = causal_conv1d_update(x,
@@ -251,7 +238,7 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
238 weight,
239 bias,
240 activation=activation)
254 - out_ref = causal_conv1d_update_ref(x,
241 + out_ref = causal_conv1d_update_ref(x_ref,
242 conv_state_ref,
243 weight,
244 bias,
@@ -260,15 +247,9 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
247 assert torch.equal(conv_state, conv_state_ref)
248 assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
249
263 - opcheck(torch.ops._C.causal_conv1d_update, (
264 - x,
265 - conv_state,
266 - weight,
267 - bias,
268 - activation in ["silu", "swish"],
269 - None,
270 - None,
271 - ))
250 + opcheck(torch.ops._C.causal_conv1d_update,
251 + (x, conv_state, weight, bias, activation
252 + in ["silu", "swish"], None, None, PAD_SLOT_ID))
253
254
255 @pytest.mark.parametrize("itype",
@@ -278,37 +259,48 @@ def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
259 @pytest.mark.parametrize("seqlen", [1, 4, 5])
260 @pytest.mark.parametrize("width", [2, 3, 4])
261 @pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
281 -def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
262 +# tests correctness in case subset of the sequences are padded
263 +@pytest.mark.parametrize("with_padding", [True, False])
264 +def test_causal_conv1d_update_with_batch_gather(with_padding, dim, width,
265 + seqlen, has_bias,
266 silu_activation, itype):
267 device = "cuda"
268 rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
269 if itype == torch.bfloat16:
270 rtol, atol = 1e-2, 5e-2
271
288 - # set )seed
272 + # set seed
273 seed_everything(0)
290 - batch = 64
274
292 - x = torch.randn(batch, dim, 1, device=device, dtype=itype)
275 + batch_size = 3
276 + padding = 5 if with_padding else 0
277 + padded_batch_size = batch_size + padding
278 + total_entries = 10 * batch_size
279
294 - total_entries = 10 * batch
280 + x = torch.randn(padded_batch_size, dim, 1, device=device, dtype=itype)
281 + x_ref = x.clone()
282 +
283 + conv_state_indices = torch.randperm(total_entries)[:batch_size].to(
284 + dtype=torch.int32, device=device)
285 + unused_states_bool = torch.ones(total_entries,
286 + dtype=torch.bool,
287 + device=device)
288 + unused_states_bool[conv_state_indices] = False
289 + padded_state_indices = torch.concat([
290 + conv_state_indices,
291 + torch.as_tensor(
292 + [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device)
293 + ],
294 + dim=0)
295 conv_state = torch.randn(total_entries,
296 dim,
297 width - 1,
298 device=device,
299 dtype=itype)
300 - conv_state_indices = torch.randperm(total_entries)[:batch].to(
301 - dtype=torch.int32, device=device)
300 + conv_state_for_padding_test = conv_state.clone()
301
303 - weight = torch.randn(dim,
304 - width,
305 - device=device,
306 - dtype=itype,
307 - requires_grad=True)
308 - if has_bias:
309 - bias = torch.randn(dim, device=device, dtype=itype, requires_grad=True)
310 - else:
311 - bias = None
302 + weight = torch.randn(dim, width, device=device, dtype=itype)
303 + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
304 conv_state_ref = conv_state[conv_state_indices, :].detach().clone()
305 activation = None if not silu_activation else "silu"
306 out = causal_conv1d_update(x,
@@ -316,45 +308,50 @@ def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
308 weight,
309 bias,
310 activation=activation,
319 - conv_state_indices=conv_state_indices)
320 - out_ref = causal_conv1d_update_ref(x,
311 + conv_state_indices=padded_state_indices,
312 + pad_slot_id=PAD_SLOT_ID)
313 + out_ref = causal_conv1d_update_ref(x_ref[:batch_size],
314 conv_state_ref,
315 weight,
316 bias,
317 activation=activation)
318
319 assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
327 - assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
320 + assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol)
321 + assert torch.equal(conv_state[unused_states_bool],
322 + conv_state_for_padding_test[unused_states_bool])
323
329 - opcheck(torch.ops._C.causal_conv1d_update, (
330 - x,
331 - conv_state,
332 - weight,
333 - bias,
334 - activation in ["silu", "swish"],
335 - None,
336 - conv_state_indices,
337 - ))
324 + opcheck(torch.ops._C.causal_conv1d_update,
325 + (x, conv_state, weight, bias, activation
326 + in ["silu", "swish"], None, padded_state_indices, PAD_SLOT_ID))
327
328
329 @pytest.mark.parametrize("itype", [torch.bfloat16])
330 @pytest.mark.parametrize("silu_activation", [True])
331 @pytest.mark.parametrize("has_bias", [True])
332 @pytest.mark.parametrize("width", [4])
344 -@pytest.mark.parametrize('seqlen',
345 - [8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 4096])
333 +@pytest.mark.parametrize(
334 + 'seqlen', [8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 2049, 4096])
335 @pytest.mark.parametrize('dim', [64, 4096])
347 -def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
348 - itype):
336 +# tests correctness in case subset of the sequences are padded
337 +@pytest.mark.parametrize('with_padding', [True, False])
338 +def test_causal_conv1d_varlen(with_padding, dim, seqlen, width, has_bias,
339 + silu_activation, itype):
340 device = "cuda"
341 + torch.cuda.empty_cache()
342 rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
343 if itype == torch.bfloat16:
344 rtol, atol = 1e-2, 5e-2
345 # set seed
346 seed_everything(0)
355 - batch = 1
347 seqlens = []
357 - nsplits = 3
348 + batch_size = 4
349 + if seqlen < 10:
350 + batch_size = 1
351 + padding = 3 if with_padding else 0
352 + padded_batch_size = batch_size + padding
353 + nsplits = padded_batch_size - 1
354 +
355 eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
356 seqlens.append(
357 torch.diff(
@@ -364,10 +361,11 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
361 assert sum(seqlens[-1]) == seqlen
362 assert all(s > 0 for s in seqlens[-1])
363
364 + total_entries = batch_size * 10
365 cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
366 cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum],
367 dim=0)
370 - x = torch.randn(batch, 4096 + dim + 64, seqlen, device=device,
368 + x = torch.randn(1, 4096 + dim + 64, seqlen, device=device,
369 dtype=itype)[:, 4096:4096 + dim, :]
370 weight = torch.randn(dim, width, device=device, dtype=itype)
371 bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
@@ -375,7 +373,7 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
373 weight_ref = weight.clone()
374 bias_ref = bias.clone() if bias is not None else None
375 activation = None if not silu_activation else "silu"
378 - final_states = torch.randn(nsplits + 1,
376 + final_states = torch.randn(total_entries,
377 dim,
378 width - 1,
379 device=x.device,
@@ -385,18 +383,27 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
383 2, (cumsum.shape[0] - 1, ),
384 dtype=torch.bool,
385 device=x.device)
388 - cache_indices = torch.randperm(cumsum.shape[0] - 1,
386 + state_indices = torch.randperm(total_entries,
387 dtype=torch.int32,
390 - device=x.device)
388 + device=x.device)[:batch_size]
389 + padded_state_indices = torch.concat([
390 + state_indices,
391 + torch.as_tensor(
392 + [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device),
393 + ],
394 + dim=-1)
395 +
396 out = causal_conv1d_fn(x.squeeze(0), weight, bias, cumsum.cuda(),
392 - cache_indices, has_initial_states, final_states,
393 - activation)
397 + padded_state_indices, has_initial_states,
398 + final_states, activation, PAD_SLOT_ID)
399 out_ref = []
400 out_ref_b = []
401
402 splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)]
403 for i in range(len(seqlens[0])):
404 x_s = [v[i].unsqueeze(0) for v in splits][0]
405 + if padded_state_indices[i] == PAD_SLOT_ID:
406 + continue
407 out_ref_b.append(
408 causal_conv1d_ref(
409 x_s,
@@ -404,21 +411,17 @@ def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
411 bias_ref,
412 activation=activation,
413 return_final_states=True,
407 - final_states_out=final_states_ref[cache_indices[i]].unsqueeze(
408 - 0),
409 - initial_states=final_states_ref[cache_indices[i]].unsqueeze(0)
410 - if has_initial_states[i] else None))
414 + final_states_out=final_states_ref[
415 + padded_state_indices[i]].unsqueeze(0),
416 + initial_states=final_states_ref[padded_state_indices[i]].
417 + unsqueeze(0) if has_initial_states[i] else None))
418 out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2))
412 - out_ref = torch.cat(out_ref, dim=0)
413 -
414 - print(f"Output max diff: {(out - out_ref).abs().max().item()}")
415 - print(f"Output mean diff: {(out - out_ref).abs().mean().item()}")
416 - print("Output state max diff"
417 - f":{(final_states - final_states_ref).abs().max()}")
418 - print("Output state mean diff"
419 - f":{(final_states - final_states_ref).abs().mean()}")
420 - assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
419 + out_ref_tensor = torch.cat(out_ref, dim=0)
420 +
421 + unpadded_out = out[:, :out_ref_tensor.shape[-1]]
422 + assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
423 assert torch.allclose(final_states, final_states_ref, rtol=rtol, atol=atol)
424 +
425 causal_conv1d_opcheck_fn(x.squeeze(0), weight, bias, cumsum.cuda(),
423 - cache_indices, has_initial_states, final_states,
424 - activation)
426 + padded_state_indices, has_initial_states,
427 + final_states, activation)
tests/kernels/test_mamba_ssm.py
+89 -35
@@ -5,6 +5,7 @@ from einops import rearrange, repeat
5
6 from tests.kernels.utils import opcheck
7 from vllm import _custom_ops as ops # noqa: F401
8 +from vllm.attention.backends.utils import PAD_SLOT_ID
9 from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
10 selective_scan_fn, selective_state_update)
11 from vllm.utils import seed_everything
@@ -174,7 +175,8 @@ def selective_scan_opcheck_fn(u,
175 cu_seq_len=None,
176 cache_indices=None,
177 has_initial_state=None,
177 - ssm_states=None):
178 + ssm_states=None,
179 + pad_slot_id=PAD_SLOT_ID):
180 """if return_last_state is True, returns (out, last_state)
181 last_state has shape (batch, dim, dstate).
182 """
@@ -203,7 +205,7 @@ def selective_scan_opcheck_fn(u,
205 # a bogus error.
206 opcheck(torch.ops._C.selective_scan_fwd,
207 (u, delta, A, B, C, D, z, delta_bias, delta_softplus, cu_seq_len,
206 - cache_indices, has_initial_state, ssm_states),
208 + cache_indices, has_initial_state, ssm_states, pad_slot_id),
209 test_utils=["test_schema", "test_faketensor"])
210
211
@@ -404,9 +406,12 @@ def test_selective_state_update(dim, dstate, has_z, itype):
406 @pytest.mark.parametrize("varBC_groups", [1, 2])
407 @pytest.mark.parametrize("is_variable_C", [True])
408 @pytest.mark.parametrize("is_variable_B", [True])
407 -def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
408 - has_D, has_z, has_delta_bias, delta_softplus,
409 - return_last_state, seqlen, itype, wtype):
409 +# tests correctness in case subset of the sequences are padded
410 +@pytest.mark.parametrize("with_padding", [False, True])
411 +def test_selective_scan_varlen(with_padding, is_variable_B, is_variable_C,
412 + varBC_groups, has_D, has_z, has_delta_bias,
413 + delta_softplus, return_last_state, seqlen,
414 + itype, wtype):
415 if varBC_groups > 1 and (not is_variable_B or not is_variable_C):
416 pytest.skip() # This config is not applicable
417 device = 'cuda'
@@ -420,18 +425,27 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
425 # set seed
426 torch.random.manual_seed(0)
427 seqlens = []
423 - nsplits = 3
428 + batch_size = 4
429 if seqlen < 10:
425 - nsplits = 0
430 + batch_size = 1
431 + padding = 3 if with_padding else 0
432 + padded_batch_size = batch_size + padding
433 +
434 + if with_padding and seqlen < padded_batch_size:
435 + pytest.skip()
436 +
437 + nsplits = padded_batch_size - 1
438 eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
439 seqlens.append(
440 torch.diff(
441 torch.cat(
442 [torch.tensor([-1]), eos_pos,
443 torch.tensor([seqlen - 1])])).tolist())
444 +
445 assert sum(seqlens[-1]) == seqlen
446 assert all(s > 0 for s in seqlens[-1])
447
448 + total_entries = batch_size * 10
449 cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
450 cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum],
451 dim=0).cuda()
@@ -462,22 +476,33 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
476 delta_ref = delta.clone()
477 out = None
478 out_ref = None
465 - prev_state_shape = (cumsum.shape[0] - 1, u.shape[0], int(A.shape[1]))
479 +
480 + prev_state_shape = (total_entries, u.shape[0], int(A.shape[1]))
481 prev_state = torch.randn(prev_state_shape,
482 device=u.device,
483 dtype=itype,
484 requires_grad=False)
485 prev_state_ref = prev_state.clone()
471 - cache_indices = torch.randperm(cumsum.shape[0] - 1,
486 + state_indices = torch.randperm(total_entries,
487 dtype=torch.int32,
473 - device=u.device)
488 + device=u.device)[:batch_size]
489 + unused_states_bool = torch.ones(total_entries,
490 + dtype=torch.bool,
491 + device=device)
492 + unused_states_bool[state_indices] = False
493 + padded_state_indices = torch.concat([
494 + state_indices,
495 + torch.as_tensor(
496 + [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device),
497 + ],
498 + dim=-1)
499
500 has_initial_state = torch.randint(0,
501 2, (cumsum.shape[0] - 1, ),
502 dtype=torch.bool,
503 device=u.device)
504 out = selective_scan_fn(u, prev_state, delta, A, B, C, D, z, delta_bias,
480 - delta_softplus, cumsum, cache_indices,
505 + delta_softplus, cumsum, padded_state_indices,
506 has_initial_state)
507 outs_ref = []
508 splits = [
@@ -486,6 +511,8 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
511 ]
512 for i in range(len(seqlens[0])):
513 u_s, delta_s, B_s, C_s, z_s = [v[i].unsqueeze(0) for v in splits]
514 + if padded_state_indices[i] == PAD_SLOT_ID:
515 + continue
516 out_ref_s, _ = selective_scan_ref(
517 u_s,
518 delta_s,
@@ -497,21 +524,22 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
524 delta_bias=delta_bias,
525 delta_softplus=delta_softplus,
526 return_last_state=return_last_state,
500 - prev_state=prev_state_ref[cache_indices[i]].unsqueeze(0)
527 + prev_state=prev_state_ref[padded_state_indices[i]].unsqueeze(0)
528 if has_initial_state[i] else None,
502 - final_state_out=prev_state_ref[cache_indices[i]].unsqueeze(0))
529 + final_state_out=prev_state_ref[padded_state_indices[i]].unsqueeze(
530 + 0))
531 outs_ref.append(out_ref_s)
504 - out_ref = torch.cat(outs_ref, dim=-1) if len(outs_ref) > 1 else outs_ref[0]
532 + out_ref = torch.cat(outs_ref, dim=-1)[0]
533
506 - print("Output diff max", (out - out_ref[0]).max())
507 - print("Output diff mean", (out - out_ref[0]).mean())
534 + unpadded_out = out[:, :out_ref[0].shape[-1]]
535 + print("Output diff max", (unpadded_out - out_ref).max())
536 + print("Output diff mean", (unpadded_out - out_ref).mean())
537 print("Output state diff max", (prev_state - prev_state_ref).max())
538 print("Output state diff mean", (prev_state - prev_state_ref).mean())
539 assert torch.allclose(prev_state, prev_state_ref, rtol=rtol, atol=atol)
511 - assert torch.allclose(out, out_ref[0], rtol=rtol, atol=atol)
512 -
540 + assert torch.allclose(unpadded_out, out_ref, rtol=rtol, atol=atol)
541 selective_scan_opcheck_fn(u, delta, A, B, C, D, z, delta_bias,
514 - delta_softplus, cumsum, cache_indices,
542 + delta_softplus, cumsum, padded_state_indices,
543 has_initial_state, prev_state)
544
545
@@ -520,7 +548,10 @@ def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
548 @pytest.mark.parametrize("has_z", [True])
549 @pytest.mark.parametrize("dstate", [16, 32, 64])
550 @pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
523 -def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
551 +# tests correctness in case subset of the sequences are padded
552 +@pytest.mark.parametrize("with_padding", [True, False])
553 +def test_selective_state_update_with_batch_indices(with_padding, dim, dstate,
554 + has_z, itype):
555 device = "cuda"
556 rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
557 if itype == torch.bfloat16:
@@ -530,21 +561,32 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
561 # set seed
562 torch.random.manual_seed(0)
563 batch_size = 3
533 -
564 + padding = 5 if with_padding else 0
565 + padded_batch_size = batch_size + padding
566 total_entries = 10 * batch_size
567 state = torch.randn(total_entries, dim, dstate, dtype=itype, device=device)
568 state_indices = torch.randperm(total_entries)[:batch_size].to(
569 dtype=torch.int32, device=device)
538 -
539 - x = torch.randn(batch_size, dim, device=device, dtype=itype)
540 - dt = torch.randn(batch_size, dim, device=device, dtype=itype)
570 + unused_states_bool = torch.ones(total_entries,
571 + dtype=torch.bool,
572 + device=device)
573 + unused_states_bool[state_indices] = False
574 + padded_state_indices = torch.concat([
575 + state_indices,
576 + torch.as_tensor(
577 + [PAD_SLOT_ID] * padding, dtype=torch.int32, device=device)
578 + ],
579 + dim=0)
580 + x = torch.randn(padded_batch_size, dim, device=device, dtype=itype)
581 + dt = torch.randn(padded_batch_size, dim, device=device, dtype=itype)
582 dt_bias = torch.rand(dim, device=device) - 4.0
583 A = -torch.rand(dim, dstate, device=device) - 1.0
543 - B = torch.randn(batch_size, dstate, device=device)
544 - C = torch.randn(batch_size, dstate, device=device)
584 + B = torch.randn(padded_batch_size, dstate, device=device)
585 + C = torch.randn(padded_batch_size, dstate, device=device)
586 D = torch.randn(dim, device=device)
587 z = torch.randn_like(x) if has_z else None
547 - state_ref = state[state_indices, :].detach().clone()
588 + state_ref = state[state_indices, :].clone()
589 + state_before = state.clone()
590 out = selective_state_update(state,
591 x,
592 dt,
@@ -555,15 +597,16 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
597 z=z,
598 dt_bias=dt_bias,
599 dt_softplus=True,
558 - state_batch_indices=state_indices)
600 + state_batch_indices=padded_state_indices,
601 + pad_slot_id=PAD_SLOT_ID)
602 out_ref = selective_state_update_ref(state_ref,
560 - x,
561 - dt,
603 + x[:batch_size],
604 + dt[:batch_size],
605 A,
563 - B,
564 - C,
606 + B[:batch_size],
607 + C[:batch_size],
608 D=D,
566 - z=z,
609 + z=z[:batch_size],
610 dt_bias=dt_bias,
611 dt_softplus=True)
612
@@ -572,11 +615,21 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
615 print("Output state diff max", (state[state_indices, :] - state_ref).max())
616 print("Output state diff mean",
617 (state[state_indices, :] - state_ref).mean())
618 + # test padded entries stay the same
619 + if with_padding:
620 + assert torch.equal(state_before[unused_states_bool],
621 + state[unused_states_bool])
622 + assert torch.equal(x[batch_size + 1:], x[batch_size + 1:])
623 + assert torch.equal(dt[batch_size + 1:], dt[batch_size + 1:])
624 + assert torch.equal(B[batch_size + 1:], B[batch_size + 1:])
625 + assert torch.equal(C[batch_size + 1:], C[batch_size + 1:])
626 +
627 + # test "real" entries
628 assert torch.allclose(state[state_indices, :],
629 state_ref,
630 rtol=rtol,
631 atol=atol)
579 - assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
632 + assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol)
633
634
635 @pytest.mark.parametrize("itype",
@@ -645,7 +698,8 @@ def test_selective_state_update_with_heads_with_batch_indices(
698 z=z,
699 dt_bias=dt_bias,
700 dt_softplus=True,
648 - state_batch_indices=state_indices)
701 + state_batch_indices=state_indices,
702 + pad_slot_id=PAD_SLOT_ID)
703 out_ref = selective_state_update_ref(state_ref,
704 x,
705 dt,
tests/models/decoder_only/language/test_jamba.py
+25
@@ -1,5 +1,6 @@
1 import pytest
2
3 +from tests.utils import multi_gpu_test
4 from vllm.sampling_params import SamplingParams
5 from vllm.worker.model_runner import _get_graph_batch_size
6
@@ -270,6 +271,30 @@ def test_state_cleanup(
271 "could be related to finished_requests_ids")
272
273
274 +@multi_gpu_test(num_gpus=2)
275 +@pytest.mark.parametrize("model", MODELS)
276 +@pytest.mark.parametrize("dtype", ["float"])
277 +@pytest.mark.parametrize("max_tokens", [64])
278 +def test_jamba_distributed_produces_identical_generation(
279 + vllm_runner, model: str, dtype: str, max_tokens: int,
280 + example_prompts) -> None:
281 +
282 + with vllm_runner(model, dtype=dtype, tensor_parallel_size=2) as vllm_model:
283 + vllm_outputs_tp_2 = vllm_model.generate_greedy(example_prompts,
284 + max_tokens)
285 +
286 + with vllm_runner(model, dtype=dtype, tensor_parallel_size=1) as vllm_model:
287 + vllm_outputs_tp_1 = vllm_model.generate_greedy(example_prompts,
288 + max_tokens)
289 +
290 + check_outputs_equal(
291 + outputs_0_lst=vllm_outputs_tp_1,
292 + outputs_1_lst=vllm_outputs_tp_2,
293 + name_0="vllm_tp_1",
294 + name_1="vllm_tp_2",
295 + )
296 +
297 +
298 @pytest.mark.parametrize("model", MODELS)
299 @pytest.mark.parametrize("dtype", ["float"])
300 def test_model_print(
vllm/_custom_ops.py
+40 -33
@@ -464,16 +464,18 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
464 cu_seq_len: Optional[torch.Tensor],
465 cache_indices: Optional[torch.Tensor],
466 has_initial_state: Optional[torch.Tensor],
467 - silu_activation: bool) -> torch.Tensor:
468 - return torch.empty_like(x)
467 + silu_activation: bool, pad_slot_id: int):
468 + return None
469
470 @register_fake("_C::causal_conv1d_update")
471 - def causal_conv1d_update_fake(
472 - x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
473 - bias_: Optional[torch.Tensor], silu_activation: bool,
474 - cache_seqlens: Optional[torch.Tensor],
475 - conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
476 - return torch.empty_like(x)
471 + def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
472 + weight: torch.Tensor,
473 + bias_: Optional[torch.Tensor],
474 + silu_activation: bool,
475 + cache_seqlens: Optional[torch.Tensor],
476 + conv_state_indices: Optional[torch.Tensor],
477 + pad_slot_id: int) -> None:
478 + return None
479
480 @register_fake("_C::selective_scan_fwd")
481 def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor,
@@ -485,7 +487,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
487 cu_seq_len: Optional[torch.Tensor],
488 cache_indices: Optional[torch.Tensor],
489 has_initial_state: Optional[torch.Tensor],
488 - ssm_states: Optional[torch.Tensor]) -> None:
490 + ssm_states: Optional[torch.Tensor],
491 + pad_slot_id: int) -> None:
492 return None
493
494
@@ -800,33 +803,37 @@ def causal_conv1d_fwd(x: torch.Tensor, weight: torch.Tensor,
803 query_start_loc: Optional[torch.Tensor],
804 cache_indices: Optional[torch.Tensor],
805 has_initial_state: Optional[torch.Tensor],
803 - silu_activation: bool) -> torch.Tensor:
804 - return torch.ops._C.causal_conv1d_fwd(x, weight, bias_, conv_states,
805 - query_start_loc, cache_indices,
806 - has_initial_state, silu_activation)
807 -
808 -
809 -def causal_conv1d_update(
810 - x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
811 - bias_: Optional[torch.Tensor], silu_activation: bool,
812 - cache_seqlens: Optional[torch.Tensor],
813 - conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
814 - return torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
815 - silu_activation, cache_seqlens,
816 - conv_state_indices)
817 -
818 -
819 -def selective_scan_fwd(
820 - u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor, B: torch.Tensor,
821 - C: torch.Tensor, D_: Optional[torch.Tensor],
822 - z_: Optional[torch.Tensor], delta_bias_: Optional[torch.Tensor],
823 - delta_softplus: bool, query_start_loc: Optional[torch.Tensor],
824 - cache_indices: Optional[torch.Tensor],
825 - has_initial_state: Optional[torch.Tensor], ssm_states: torch.Tensor):
806 + silu_activation: bool, pad_slot_id: int):
807 + torch.ops._C.causal_conv1d_fwd(x, weight, bias_, conv_states,
808 + query_start_loc, cache_indices,
809 + has_initial_state, silu_activation,
810 + pad_slot_id)
811 +
812 +
813 +def causal_conv1d_update(x: torch.Tensor, conv_state: torch.Tensor,
814 + weight: torch.Tensor, bias_: Optional[torch.Tensor],
815 + silu_activation: bool,
816 + cache_seqlens: Optional[torch.Tensor],
817 + conv_state_indices: Optional[torch.Tensor],
818 + pad_slot_id: int):
819 + torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
820 + silu_activation, cache_seqlens,
821 + conv_state_indices, pad_slot_id)
822 +
823 +
824 +def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
825 + B: torch.Tensor, C: torch.Tensor,
826 + D_: Optional[torch.Tensor], z_: Optional[torch.Tensor],
827 + delta_bias_: Optional[torch.Tensor],
828 + delta_softplus: bool,
829 + query_start_loc: Optional[torch.Tensor],
830 + cache_indices: Optional[torch.Tensor],
831 + has_initial_state: Optional[torch.Tensor],
832 + ssm_states: torch.Tensor, pad_slot_id: int):
833 torch.ops._C.selective_scan_fwd(u, delta, A, B, C, D_, z_, delta_bias_,
834 delta_softplus, query_start_loc,
835 cache_indices, has_initial_state,
829 - ssm_states)
836 + ssm_states, pad_slot_id)
837
838
839 # moe
vllm/model_executor/layers/mamba/ops/causal_conv1d.py
+33 -20
@@ -6,18 +6,18 @@ from typing import Optional
6 import torch
7
8 from vllm import _custom_ops as ops
9 +from vllm.attention.backends.utils import PAD_SLOT_ID
10
11
11 -def causal_conv1d_fn(
12 - x: torch.Tensor,
13 - weight: torch.Tensor,
14 - bias: Optional[torch.Tensor] = None,
15 - query_start_loc: Optional[torch.Tensor] = None,
16 - cache_indices: Optional[torch.Tensor] = None,
17 - has_initial_state: Optional[torch.Tensor] = None,
18 - conv_states: Optional[torch.Tensor] = None,
19 - activation: Optional[str] = "silu",
20 -):
12 +def causal_conv1d_fn(x: torch.Tensor,
13 + weight: torch.Tensor,
14 + bias: Optional[torch.Tensor] = None,
15 + query_start_loc: Optional[torch.Tensor] = None,
16 + cache_indices: Optional[torch.Tensor] = None,
17 + has_initial_state: Optional[torch.Tensor] = None,
18 + conv_states: Optional[torch.Tensor] = None,
19 + activation: Optional[str] = "silu",
20 + pad_slot_id: int = PAD_SLOT_ID):
21 """
22 x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen
23 sequences are concatenated from left to right for varlen
@@ -37,6 +37,13 @@ def causal_conv1d_fn(
37 conv_states: (...,dim,width - 1) itype
38 updated inplace if provided
39 activation: either None or "silu" or "swish"
40 + pad_slot_id: int
41 + if cache_indices is passed, lets the kernel identify padded
42 + entries that will not be processed,
43 + for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
44 + in this case, the kernel will not process entries at
45 + indices 0 and 3
46 +
47
48 out: (batch, dim, seqlen)
49 """
@@ -46,10 +53,10 @@ def causal_conv1d_fn(
53 x = x.contiguous()
54 bias = bias.contiguous() if bias is not None else None
55
49 - out = ops.causal_conv1d_fwd(x, weight, bias, conv_states, query_start_loc,
50 - cache_indices, has_initial_state, activation
51 - in ["silu", "swish"])
52 - return out
56 + ops.causal_conv1d_fwd(x, weight, bias, conv_states, query_start_loc,
57 + cache_indices, has_initial_state, activation
58 + in ["silu", "swish"], pad_slot_id)
59 + return x
60
61
62 def causal_conv1d_update(x: torch.Tensor,
@@ -58,7 +65,8 @@ def causal_conv1d_update(x: torch.Tensor,
65 bias: Optional[torch.Tensor] = None,
66 activation: Optional[str] = None,
67 cache_seqlens: Optional[torch.Tensor] = None,
61 - conv_state_indices: Optional[torch.Tensor] = None):
68 + conv_state_indices: Optional[torch.Tensor] = None,
69 + pad_slot_id: int = PAD_SLOT_ID):
70 """
71 x: (batch, dim) or (batch, dim, seqlen)
72 conv_state: (batch, dim, state_len), where state_len >= width - 1
@@ -73,7 +81,12 @@ def causal_conv1d_update(x: torch.Tensor,
81 If not None, the conv_state is a larger tensor along the batch dim,
82 and we are selecting the batch coords specified by conv_state_indices.
83 Useful for a continuous batching scenario.
76 -
84 + pad_slot_id: int
85 + if cache_indices is passed, lets the kernel identify padded
86 + entries that will not be processed,
87 + for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id]
88 + in this case, the kernel will not process entries at
89 + indices 0 and 3
90 out: (batch, dim) or (batch, dim, seqlen)
91 """
92 if activation not in [None, "silu", "swish"]:
@@ -82,8 +95,8 @@ def causal_conv1d_update(x: torch.Tensor,
95 unsqueeze = x.dim() == 2
96 if unsqueeze:
97 x = x.unsqueeze(-1)
85 - out = ops.causal_conv1d_update(x, conv_state, weight, bias, activation_val,
86 - cache_seqlens, conv_state_indices)
98 + ops.causal_conv1d_update(x, conv_state, weight, bias, activation_val,
99 + cache_seqlens, conv_state_indices, pad_slot_id)
100 if unsqueeze:
88 - out = out.squeeze(-1)
89 - return out
101 + x = x.squeeze(-1)
102 + return x
vllm/model_executor/layers/mamba/ops/mamba_ssm.py
+43 -27
@@ -1,14 +1,13 @@
1 # Copyright (c) 2024, Tri Dao, Albert Gu.
2 # Adapted from https://github.com/state-spaces/mamba/blob/main/mamba_ssm/ops/triton/selective_state_update.py
3
4 -from typing import Tuple
5 -
4 import torch
5 import triton
6 import triton.language as tl
7 from packaging import version
8
9 from vllm import _custom_ops as ops
10 +from vllm.attention.backends.utils import PAD_SLOT_ID
11
12 TRITON3 = version.parse(triton.__version__) >= version.parse("3.0.0")
13
@@ -50,6 +49,7 @@ def _selective_scan_update_kernel(
49 z_ptr,
50 out_ptr,
51 state_batch_indices_ptr,
52 + pad_slot_id,
53 # Matrix dimensions
54 batch,
55 nheads,
@@ -143,10 +143,11 @@ def _selective_scan_update_kernel(
143 if HAS_Z:
144 z_ptrs = z_ptr + offs_m * stride_z_dim
145 out_ptrs = out_ptr + offs_m * stride_out_dim
146 + mask = (offs_m[:, None] < dim) & (offs_n[None, :] < dstate)
147 + if HAS_STATE_BATCH_INDICES:
148 + mask &= (state_batch_idx != pad_slot_id)
149 + state = tl.load(state_ptrs, mask=mask, other=0.0)
150
147 - state = tl.load(state_ptrs,
148 - mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate),
149 - other=0.0)
151 x = tl.load(x_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
152 if not TIE_HDIM:
153 dt = tl.load(dt_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
@@ -177,9 +178,11 @@ def _selective_scan_update_kernel(
178
179 dB = B[None, :] * dt[:, None] if not TIE_HDIM else B * dt
180 state = state * dA + dB * x[:, None]
180 - tl.store(state_ptrs,
181 - state,
182 - mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate))
181 +
182 + mask = (offs_m[:, None] < dim) & (offs_n[None, :] < dstate)
183 + if HAS_STATE_BATCH_INDICES:
184 + mask &= (state_batch_idx != pad_slot_id)
185 + tl.store(state_ptrs, state, mask=mask)
186 out = tl.sum(state * C[None, :], axis=1)
187 if HAS_D:
188 out += x * D
@@ -198,7 +201,8 @@ def selective_state_update(state,
201 z=None,
202 dt_bias=None,
203 dt_softplus=False,
201 - state_batch_indices=None):
204 + state_batch_indices=None,
205 + pad_slot_id=PAD_SLOT_ID):
206 """
207 Argument:
208 state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
@@ -210,6 +214,12 @@ def selective_state_update(state,
214 D: (dim,) or (nheads, dim)
215 z: (batch, dim) or (batch, nheads, dim)
216 dt_bias: (dim,) or (nheads, dim)
217 + pad_slot_id: int
218 + if cache_indices is passed, lets the kernel identify padded
219 + entries that will not be processed,
220 + for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
221 + in this case, the kernel will not process entries at
222 + indices 0 and 3
223 Return:
224 out: (batch, dim) or (batch, nheads, dim)
225 """
@@ -276,6 +286,7 @@ def selective_state_update(state,
286 z,
287 out,
288 state_batch_indices,
289 + pad_slot_id,
290 batch,
291 nheads,
292 dim,
@@ -319,22 +330,25 @@ def selective_state_update(state,
330 return out
331
332
322 -def selective_scan_fn(
323 - u,
324 - ssm_states,
325 - delta,
326 - A,
327 - B,
328 - C,
329 - D=None,
330 - z=None,
331 - delta_bias=None,
332 - delta_softplus=False,
333 - query_start_loc=None,
334 - cache_indices=None,
335 - has_initial_state=None) -> Tuple[torch.Tensor, torch.Tensor]:
333 +def selective_scan_fn(u,
334 + ssm_states,
335 + delta,
336 + A,
337 + B,
338 + C,
339 + D=None,
340 + z=None,
341 + delta_bias=None,
342 + delta_softplus=False,
343 + query_start_loc=None,
344 + cache_indices=None,
345 + has_initial_state=None,
346 + pad_slot_id=PAD_SLOT_ID) -> torch.Tensor:
347 """
348 u: (dim, total_length) for varlen or (batch, dim, seqlen)
349 + applies changes in place.
350 + ssm_states: (batch, dim, dstate) or (batch, nheads, dim, dstate)
351 + applies changes in place.
352 delta: (dim, total_length) for varlen or (batch, dim, seqlen)
353 A: (dim, dstate)
354 B: (ngroups, dstate, total_length) for varlen or
@@ -357,12 +371,14 @@ def selective_scan_fn(
371 indicate if the ssm_state at the corresponding index should be
372 used as initial state. Not providing argument assumes
373 there's no initial state
360 -
374 + pad_slot_id: int
375 + if cache_indices is passed, lets the kernel identify padding entries
376 + that will not be processed,
377 + for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id]
378 + in this case, the kernel will not process entries at indices 0 and 3
379 returns
380 output: (dim, total_length) for varlen or (batch, dim, seqlen)
381 supports inplace replacement
364 - last_state has shape (batch, dim, dstate).
365 - supports inplace replacement if ssm_state was provided
382 """
383 if u.stride(-1) != 1:
384 u = u.contiguous()
@@ -387,7 +403,7 @@ def selective_scan_fn(
403
404 ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias, delta_softplus,
405 query_start_loc, cache_indices, has_initial_state,
390 - ssm_states)
406 + ssm_states, pad_slot_id)
407
408 if z is None:
409 return delta # output written inplace to delta
vllm/model_executor/models/jamba.py
+31 -40
@@ -1,6 +1,5 @@
1 # coding=utf-8
2 """Inference-only Jamba model."""
3 -from dataclasses import dataclass
3 from typing import Iterable, List, Optional, Tuple
4
5 import torch
@@ -29,7 +28,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
28 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
29 from vllm.model_executor.model_loader.weight_utils import (
30 composed_weight_loader, default_weight_loader, sharded_weight_loader)
32 -from vllm.model_executor.models.mamba_cache import MambaCacheManager
31 +from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
32 + MambaCacheParams)
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.model_executor.utils import set_weight_attrs
35 from vllm.sequence import IntermediateTensors
@@ -41,13 +41,6 @@ from .interfaces import HasInnerState, SupportsLoRA
41 KVCache = Tuple[torch.Tensor, torch.Tensor]
42
43
44 -@dataclass
45 -class MambaCacheParams:
46 - is_prompt: bool = False
47 - conv_state: torch.Tensor = torch.Tensor()
48 - ssm_state: torch.Tensor = torch.Tensor()
49 -
50 -
44 # Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
45 class JambaMambaMixer(nn.Module):
46 """
@@ -60,10 +53,9 @@ class JambaMambaMixer(nn.Module):
53 **selective** state spaces)
54 """
55
63 - def __init__(self, config: JambaConfig, layer_idx):
56 + def __init__(self, config: JambaConfig):
57 super().__init__()
58 self.config = config
66 - self.layer_idx = layer_idx
59 self.hidden_size = config.hidden_size
60 self.ssm_state_size = config.mamba_d_state
61 self.conv_kernel_size = config.mamba_d_conv
@@ -129,8 +121,8 @@ class JambaMambaMixer(nn.Module):
121 eps=config.rms_norm_eps)
122
123 def forward(self, hidden_states: torch.Tensor,
132 - attn_metadata: AttentionMetadata, conv_state: torch.Tensor,
133 - ssm_state: torch.Tensor):
124 + attn_metadata: AttentionMetadata,
125 + mamba_cache_params: MambaCacheParams):
126
127 # 1. Gated MLP's linear projection
128 projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
@@ -153,17 +145,18 @@ class JambaMambaMixer(nn.Module):
145 conv_weights,
146 self.conv1d.bias,
147 activation=self.activation,
156 - conv_states=conv_state,
148 + conv_states=mamba_cache_params.conv_state,
149 has_initial_state=attn_metadata.context_lens_tensor > 0,
150 + cache_indices=mamba_cache_params.state_indices_tensor,
151 query_start_loc=attn_metadata.query_start_loc)
152 else:
153 hidden_states = causal_conv1d_update(
154 hidden_states.transpose(0, 1),
162 - conv_state,
155 + mamba_cache_params.conv_state,
156 conv_weights,
157 self.conv1d.bias,
158 self.activation,
166 - )
159 + conv_state_indices=mamba_cache_params.state_indices_tensor)
160 hidden_states = hidden_states.transpose(0, 1)
161
162 # 3. State Space Model sequence transformation
@@ -188,7 +181,7 @@ class JambaMambaMixer(nn.Module):
181 and attn_metadata.context_lens_tensor is not None:
182 scan_outputs = selective_scan_fn(
183 hidden_states,
191 - ssm_state,
184 + mamba_cache_params.ssm_state,
185 discrete_time_step,
186 self.A,
187 B.transpose(-2, -1),
@@ -197,11 +190,12 @@ class JambaMambaMixer(nn.Module):
190 gate,
191 time_proj_bias,
192 delta_softplus=True,
193 + cache_indices=mamba_cache_params.state_indices_tensor,
194 has_initial_state=attn_metadata.context_lens_tensor > 0,
195 query_start_loc=attn_metadata.query_start_loc)
196 else:
197 scan_outputs = selective_state_update(
204 - ssm_state,
198 + mamba_cache_params.ssm_state,
199 hidden_states.transpose(0, 1),
200 discrete_time_step.transpose(0, 1),
201 self.A,
@@ -211,7 +205,7 @@ class JambaMambaMixer(nn.Module):
205 gate.transpose(0, 1),
206 time_proj_bias,
207 dt_softplus=True,
214 - )
208 + state_batch_indices=mamba_cache_params.state_indices_tensor)
209 scan_outputs = scan_outputs.transpose(0, 1)
210
211 # 4. Final linear projection
@@ -292,7 +286,7 @@ class JambaMambaDecoderLayer(nn.Module):
286 super().__init__()
287 self.layer_idx = layer_idx
288 self.config = config
295 - self.mamba = JambaMambaMixer(config, layer_idx)
289 + self.mamba = JambaMambaMixer(config)
290
291 num_experts = config.layers_num_experts[layer_idx]
292 ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
@@ -307,8 +301,7 @@ class JambaMambaDecoderLayer(nn.Module):
301 hidden_states: torch.Tensor,
302 attn_metadata: AttentionMetadata,
303 residual: Optional[torch.Tensor],
310 - conv_state: torch.Tensor,
311 - ssm_state: torch.Tensor,
304 + mamba_cache_params: MambaCacheParams,
305 **kwargs,
306 ):
307 if residual is None:
@@ -318,8 +311,8 @@ class JambaMambaDecoderLayer(nn.Module):
311 hidden_states, residual = self.input_layernorm(
312 hidden_states, residual)
313
321 - hidden_states = self.mamba(hidden_states, attn_metadata, conv_state,
322 - ssm_state)
314 + hidden_states = self.mamba(hidden_states, attn_metadata,
315 + mamba_cache_params)
316 # Fully Connected
317 hidden_states, residual = self.pre_ff_layernorm(
318 hidden_states, residual)
@@ -476,17 +469,14 @@ class JambaModel(nn.Module):
469 positions: torch.Tensor,
470 kv_caches: List[torch.Tensor],
471 attn_metadata: AttentionMetadata,
479 - conv_state: torch.Tensor,
480 - ssm_state: torch.Tensor,
472 + mamba_cache_params: MambaCacheParams,
473 ) -> torch.Tensor:
474 hidden_states = self.embed_tokens(input_ids)
475 residual = None
484 -
476 for i in range(len(self.layers)):
477 layer = self.layers[i]
478 kv_cache = None
488 - current_ssm_state = None
489 - current_conv_state = None
479 + layer_mamba_cache_params = None
480 if isinstance(layer, JambaAttentionDecoderLayer):
481 kv_cache = kv_caches[(i - self.config.attn_layer_offset) //
482 self.config.attn_layer_period]
@@ -494,8 +484,8 @@ class JambaModel(nn.Module):
484 current_state_layer = i - (1 +
485 (i - self.config.attn_layer_offset)
486 // self.config.attn_layer_period)
497 - current_ssm_state = ssm_state[current_state_layer]
498 - current_conv_state = conv_state[current_state_layer]
487 + layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
488 + current_state_layer)
489
490 hidden_states, residual = layer(
491 positions=positions,
@@ -503,9 +493,7 @@ class JambaModel(nn.Module):
493 kv_cache=kv_cache,
494 attn_metadata=attn_metadata,
495 residual=residual,
506 - conv_state=current_conv_state,
507 - ssm_state=current_ssm_state,
508 - )
496 + mamba_cache_params=layer_mamba_cache_params)
497 hidden_states, _ = self.final_layernorm(hidden_states, residual)
498 return hidden_states
499
@@ -588,13 +576,16 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
576 self.mamba_cache = MambaCacheManager(
577 self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
578 *self._get_mamba_cache_shape())
591 -
592 - mamba_cache_tensors = self.mamba_cache.current_run_tensors(
593 - input_ids, attn_metadata, **kwargs)
594 -
579 + (
580 + mamba_cache_tensors,
581 + state_indices_tensor,
582 + ) = self.mamba_cache.current_run_tensors(input_ids, attn_metadata,
583 + **kwargs)
584 + mamba_cache_params = MambaCacheParams(mamba_cache_tensors[0],
585 + mamba_cache_tensors[1],
586 + state_indices_tensor)
587 hidden_states = self.model(input_ids, positions, kv_caches,
596 - attn_metadata, mamba_cache_tensors[0],
597 - mamba_cache_tensors[1])
588 + attn_metadata, mamba_cache_params)
589 return hidden_states
590
591 def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
vllm/model_executor/models/mamba.py
+28 -25
@@ -27,7 +27,8 @@ from vllm.model_executor.model_loader.weight_utils import (
27 composed_weight_loader, default_weight_loader, sharded_weight_loader)
28 from vllm.model_executor.models.interfaces import (HasInnerState,
29 IsAttentionFree)
30 -from vllm.model_executor.models.mamba_cache import MambaCacheManager
30 +from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
31 + MambaCacheParams)
32 from vllm.model_executor.sampling_metadata import SamplingMetadata
33 from vllm.model_executor.utils import set_weight_attrs
34 from vllm.sequence import IntermediateTensors
@@ -110,8 +111,8 @@ class MambaMixer(nn.Module):
111 self.activation = config.hidden_act
112
113 def forward(self, hidden_states: torch.Tensor,
113 - attn_metadata: AttentionMetadata, conv_state: torch.Tensor,
114 - ssm_state: torch.Tensor):
114 + attn_metadata: AttentionMetadata,
115 + mamba_cache_params: MambaCacheParams):
116
117 # 1. Gated MLP's linear projection
118 projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
@@ -134,17 +135,18 @@ class MambaMixer(nn.Module):
135 conv_weights,
136 self.conv1d.bias,
137 activation=self.activation,
137 - conv_states=conv_state,
138 + conv_states=mamba_cache_params.conv_state,
139 has_initial_state=attn_metadata.context_lens_tensor > 0,
140 + cache_indices=mamba_cache_params.state_indices_tensor,
141 query_start_loc=attn_metadata.query_start_loc)
142 else:
143 hidden_states = causal_conv1d_update(
144 hidden_states.transpose(0, 1),
143 - conv_state,
145 + mamba_cache_params.conv_state,
146 conv_weights,
147 self.conv1d.bias,
148 self.activation,
147 - )
149 + conv_state_indices=mamba_cache_params.state_indices_tensor)
150 hidden_states = hidden_states.transpose(0, 1)
151
152 # 3. State Space Model sequence transformation
@@ -168,7 +170,7 @@ class MambaMixer(nn.Module):
170 and attn_metadata.context_lens_tensor is not None:
171 scan_outputs = selective_scan_fn(
172 hidden_states,
171 - ssm_state,
173 + mamba_cache_params.ssm_state,
174 discrete_time_step,
175 self.A,
176 B.transpose(-2, -1),
@@ -177,11 +179,12 @@ class MambaMixer(nn.Module):
179 gate,
180 time_proj_bias,
181 delta_softplus=True,
182 + cache_indices=mamba_cache_params.state_indices_tensor,
183 has_initial_state=attn_metadata.context_lens_tensor > 0,
184 query_start_loc=attn_metadata.query_start_loc)
185 else:
186 scan_outputs = selective_state_update(
184 - ssm_state,
187 + mamba_cache_params.ssm_state,
188 hidden_states.transpose(0, 1),
189 discrete_time_step.transpose(0, 1),
190 self.A,
@@ -191,7 +194,7 @@ class MambaMixer(nn.Module):
194 gate.transpose(0, 1),
195 time_proj_bias,
196 dt_softplus=True,
194 - )
197 + state_batch_indices=mamba_cache_params.state_indices_tensor)
198 scan_outputs = scan_outputs.transpose(0, 1)
199
200 # 4. Final linear projection
@@ -221,8 +224,7 @@ class MambaDecoderLayer(nn.Module):
224 hidden_states: torch.Tensor,
225 attn_metadata: AttentionMetadata,
226 residual: Optional[torch.Tensor],
224 - conv_state: torch.Tensor,
225 - ssm_state: torch.Tensor,
227 + mamba_cache_params: MambaCacheParams,
228 **kwargs,
229 ):
230 if residual is None:
@@ -231,8 +233,8 @@ class MambaDecoderLayer(nn.Module):
233 else:
234 hidden_states, residual = self.norm(hidden_states, residual)
235
234 - hidden_states = self.mixer(hidden_states, attn_metadata, conv_state,
235 - ssm_state)
236 + hidden_states = self.mixer(hidden_states, attn_metadata,
237 + mamba_cache_params)
238 return hidden_states, residual
239
240
@@ -275,25 +277,20 @@ class MambaModel(nn.Module):
277 input_ids: torch.Tensor,
278 positions: torch.Tensor,
279 attn_metadata: AttentionMetadata,
278 - conv_state: torch.Tensor,
279 - ssm_state: torch.Tensor,
280 + mamba_cache_params: MambaCacheParams,
281 ) -> torch.Tensor:
282 +
283 hidden_states = self.embeddings(input_ids)
284 residual = None
285
286 for i in range(len(self.layers)):
287 layer = self.layers[i]
286 - current_ssm_state = ssm_state[i]
287 - current_conv_state = conv_state[i]
288 -
288 hidden_states, residual = layer(
289 positions=positions,
290 hidden_states=hidden_states,
291 attn_metadata=attn_metadata,
292 residual=residual,
294 - conv_state=current_conv_state,
295 - ssm_state=current_ssm_state,
296 - )
293 + mamba_cache_params=mamba_cache_params.at_layer_idx(i))
294 hidden_states, _ = self.norm_f(hidden_states, residual)
295
296 return hidden_states
@@ -347,12 +344,18 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
344 self.lm_head.weight.dtype, self.config.num_hidden_layers,
345 max_batch_size, *self._get_mamba_cache_shape())
346
350 - mamba_cache_tensors = self.mamba_cache.current_run_tensors(
351 - input_ids, attn_metadata, **kwargs)
347 + (
348 + mamba_cache_tensors,
349 + state_indices_tensor,
350 + ) = self.mamba_cache.current_run_tensors(input_ids, attn_metadata,
351 + **kwargs)
352 +
353 + mamba_cache_params = MambaCacheParams(mamba_cache_tensors[0],
354 + mamba_cache_tensors[1],
355 + state_indices_tensor)
356
357 hidden_states = self.backbone(input_ids, positions, attn_metadata,
354 - mamba_cache_tensors[0],
355 - mamba_cache_tensors[1])
358 + mamba_cache_params)
359
360 return hidden_states
361
vllm/model_executor/models/mamba_cache.py
+61 -125
@@ -1,8 +1,22 @@
1 -from typing import Dict, List, Optional
1 +from dataclasses import dataclass
2 +from typing import Dict, List
3
4 import torch
5
6 from vllm.attention.backends.abstract import AttentionMetadata
7 +from vllm.attention.backends.utils import PAD_SLOT_ID
8 +
9 +
10 +@dataclass
11 +class MambaCacheParams:
12 + conv_state: torch.Tensor = torch.Tensor()
13 + ssm_state: torch.Tensor = torch.Tensor()
14 + state_indices_tensor: torch.Tensor = torch.Tensor()
15 +
16 + def at_layer_idx(self, layer_idx):
17 + return MambaCacheParams(self.conv_state[layer_idx],
18 + self.ssm_state[layer_idx],
19 + self.state_indices_tensor)
20
21
22 class MambaCacheManager:
@@ -24,6 +38,7 @@ class MambaCacheManager:
38 # Maps between the request id and a dict that maps between the seq_id
39 # and its index inside the self.mamba_cache
40 self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
41 + self.free_cache_indices = list(range(max_batch_size))
42
43 def current_run_tensors(self, input_ids: torch.Tensor,
44 attn_metadata: AttentionMetadata, **kwargs):
@@ -36,30 +51,43 @@ class MambaCacheManager:
51 finished_requests_ids = kwargs["finished_requests_ids"]
52
53 self._release_finished_requests(finished_requests_ids)
39 - mamba_cache_tensors = self._prepare_current_run_mamba_cache(
54 + state_indices = self._prepare_current_run_mamba_cache(
55 request_ids_to_seq_ids, finished_requests_ids)
56
57 + state_indices_tensor = torch.as_tensor(state_indices,
58 + dtype=torch.int32,
59 + device="cuda")
60 + mamba_cache_tensors = self.mamba_cache
61 +
62 else:
63 # CUDA graph capturing runs
44 - mamba_cache_tensors = kwargs["seqlen_agnostic_capture_inputs"]
64 + (mamba_cache_tensors,
65 + state_indices_tensor) = kwargs["seqlen_agnostic_capture_inputs"]
66
46 - return mamba_cache_tensors
67 + return (mamba_cache_tensors, state_indices_tensor)
68
69 def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
70 """
50 - Copy the relevant Mamba cache into the CUDA graph input buffer
51 - that was provided during the capture runs
52 - (JambaForCausalLM.mamba_gc_cache_buffer).
71 + Copy the relevant state_indices into the CUDA graph input buffer
72 """
73 assert all(
74 key in kwargs
75 for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
76 finished_requests_ids = kwargs["finished_requests_ids"]
77 request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
78 + assert "seqlen_agnostic_capture_inputs" in input_buffers
79 + _, input_state_indices_buffer = input_buffers[
80 + "seqlen_agnostic_capture_inputs"]
81
82 self._release_finished_requests(finished_requests_ids)
61 - self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
62 - finished_requests_ids)
83 + state_indices = self._prepare_current_run_mamba_cache(
84 + request_ids_to_seq_ids, finished_requests_ids)
85 + cuda_graph_pad_len = input_state_indices_buffer.shape[0] - len(
86 + state_indices)
87 + state_indices.extend([PAD_SLOT_ID] * cuda_graph_pad_len)
88 +
89 + input_state_indices_buffer.copy_(
90 + torch.as_tensor(state_indices, dtype=torch.int32, device="cuda"))
91
92 def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
93 """
@@ -67,13 +95,10 @@ class MambaCacheManager:
95 The buffer is used to maintain the Mamba Cache during the CUDA graph
96 replay runs.
97 """
70 - return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache)
71 -
72 - def _swap_mamba_cache(self, from_index: int, to_index: int):
73 - assert len(self.mamba_cache) > 0
74 - for cache_t in self.mamba_cache:
75 - cache_t[:, [to_index,from_index]] = \
76 - cache_t[:, [from_index,to_index]]
98 + state_indices_tensor = torch.as_tensor([PAD_SLOT_ID] * batch_size,
99 + dtype=torch.int32,
100 + device="cuda")
101 + return (self.mamba_cache, state_indices_tensor)
102
103 def _copy_mamba_cache(self, from_index: int, to_index: int):
104 assert len(self.mamba_cache) > 0
@@ -81,142 +106,53 @@ class MambaCacheManager:
106 cache_t[:, to_index].copy_(cache_t[:, from_index],
107 non_blocking=True)
108
84 - def _move_out_if_already_occupied(self, index: int,
85 - all_occupied_indices: List[int]):
86 - if index in all_occupied_indices:
87 - first_free_index = self._first_free_index_in_mamba_cache()
88 - # In case occupied, move the occupied to a new empty block
89 - self._move_cache_index_and_mappings(from_index=index,
90 - to_index=first_free_index)
91 -
92 - def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str,
93 - seq_id: int,
94 - destination_index: int):
109 + def _assign_seq_id_to_cache_index(self, cur_rid: str, seq_id: int,
110 + finished_requests_ids) -> int:
111 """
112 Assign (req_id,seq_id) pair to a `destination_index` index, if
113 already occupied, move the occupying index to a free index.
114 """
99 - all_occupied_indices = self._get_all_occupied_indices()
100 - if cur_rid not in self.mamba_cache_indices_mapping:
101 - self._move_out_if_already_occupied(
102 - index=destination_index,
103 - all_occupied_indices=all_occupied_indices)
115 + if cur_rid in finished_requests_ids:
116 + # set as pad, do not allocate destination index
117 + return PAD_SLOT_ID
118 + elif cur_rid not in self.mamba_cache_indices_mapping:
119 + destination_index = self.free_cache_indices.pop()
120 self.mamba_cache_indices_mapping[cur_rid] = {
121 seq_id: destination_index
122 }
123 + return destination_index
124 elif seq_id not in (seq_ids2indices :=
125 self.mamba_cache_indices_mapping[cur_rid]):
126 # parallel sampling , where n > 1, assume prefill have
110 - # already happened now we only need to copy the already
127 + # already happened, so we copy the
128 # existing cache into the siblings seq_ids caches
112 - self._move_out_if_already_occupied(
113 - index=destination_index,
114 - all_occupied_indices=all_occupied_indices)
115 - index_exists = list(seq_ids2indices.values())[0]
129 + index_exists = next(iter(seq_ids2indices.values()))
130 # case of decoding n>1, copy prefill cache to decoding indices
131 + destination_index = self.free_cache_indices.pop()
132 self._copy_mamba_cache(from_index=index_exists,
133 to_index=destination_index)
134 self.mamba_cache_indices_mapping[cur_rid][
135 seq_id] = destination_index
136 + return destination_index
137 else:
138 # already exists
123 - cache_index_already_exists = self.mamba_cache_indices_mapping[
124 - cur_rid][seq_id]
125 - if cache_index_already_exists != destination_index:
126 - # In case the seq id already exists but not in
127 - # the right destination, swap it with what's occupying it
128 - self._swap_pair_indices_and_mappings(
129 - from_index=cache_index_already_exists,
130 - to_index=destination_index)
139 + return self.mamba_cache_indices_mapping[cur_rid][seq_id]
140
141 def _prepare_current_run_mamba_cache(
142 self, request_ids_to_seq_ids: Dict[str, list[int]],
134 - finished_requests_ids: List[str]):
135 - running_indices = []
136 - request_ids_to_seq_ids_flatten = [
137 - (req_id, seq_id)
143 + finished_requests_ids: List[str]) -> List[int]:
144 + return [
145 + self._assign_seq_id_to_cache_index(req_id, seq_id,
146 + finished_requests_ids)
147 for req_id, seq_ids in request_ids_to_seq_ids.items()
148 for seq_id in seq_ids
149 ]
141 - batch_size = len(request_ids_to_seq_ids_flatten)
142 - for dest_index, (request_id,
143 - seq_id) in enumerate(request_ids_to_seq_ids_flatten):
144 - if request_id in finished_requests_ids:
145 - # Do not allocate cache index for requests that run
146 - # and finish right after
147 - continue
148 - self._assign_seq_id_to_mamba_cache_in_specific_dest(
149 - request_id, seq_id, dest_index)
150 - running_indices.append(dest_index)
151 -
152 - self._clean_up_first_bs_blocks(batch_size, running_indices)
153 - conv_state = self.mamba_cache[0][:, :batch_size]
154 - temporal_state = self.mamba_cache[1][:, :batch_size]
155 -
156 - return (conv_state, temporal_state)
157 -
158 - def _get_all_occupied_indices(self):
159 - return [
160 - cache_idx
161 - for seq_ids2indices in self.mamba_cache_indices_mapping.values()
162 - for cache_idx in seq_ids2indices.values()
163 - ]
164 -
165 - def _clean_up_first_bs_blocks(self, batch_size: int,
166 - indices_for_current_run: List[int]):
167 - # move out all of the occupied but currently not running blocks
168 - # outside of the first n blocks
169 - destination_indices = range(batch_size)
170 - max_possible_batch_size = self.mamba_cache[0].shape[1]
171 - for destination_index in destination_indices:
172 - if destination_index in self._get_all_occupied_indices() and \
173 - destination_index not in indices_for_current_run:
174 - # move not running indices outside of the batch
175 - all_other_indices = list(
176 - range(batch_size, max_possible_batch_size))
177 - first_avail_index = self._first_free_index_in_mamba_cache(
178 - all_other_indices)
179 - self._swap_indices(from_index=destination_index,
180 - to_index=first_avail_index)
181 -
182 - def _move_cache_index_and_mappings(self, from_index: int, to_index: int):
183 - self._copy_mamba_cache(from_index=from_index, to_index=to_index)
184 - self._update_mapping_index(from_index=from_index, to_index=to_index)
185 -
186 - def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int):
187 - self._swap_mamba_cache(from_index=from_index, to_index=to_index)
188 - self._swap_mapping_index(from_index=from_index, to_index=to_index)
189 -
190 - def _swap_mapping_index(self, from_index: int, to_index: int):
191 - for seq_ids2index in self.mamba_cache_indices_mapping.values():
192 - for seq_id, index in seq_ids2index.items():
193 - if from_index == index:
194 - seq_ids2index.update({seq_id: to_index})
195 - elif to_index == index:
196 - seq_ids2index.update({seq_id: from_index})
197 -
198 - def _update_mapping_index(self, from_index: int, to_index: int):
199 - for seq_ids2index in self.mamba_cache_indices_mapping.values():
200 - for seq_id, index in seq_ids2index.items():
201 - if from_index == index:
202 - seq_ids2index.update({seq_id: to_index})
203 - return
150
151 def _release_finished_requests(self,
152 finished_seq_groups_req_ids: List[str]):
153 for req_id in finished_seq_groups_req_ids:
154 if req_id in self.mamba_cache_indices_mapping:
155 + for seq_id in self.mamba_cache_indices_mapping[req_id]:
156 + self.free_cache_indices.append(
157 + self.mamba_cache_indices_mapping[req_id][seq_id])
158 self.mamba_cache_indices_mapping.pop(req_id)
210 -
211 - def _first_free_index_in_mamba_cache(
212 - self, indices_range: Optional[List[int]] = None) -> int:
213 - assert self.mamba_cache is not None
214 - if indices_range is None:
215 - max_possible_batch_size = self.mamba_cache[0].shape[1]
216 - indices_range = list(range(max_possible_batch_size))
217 - all_occupied_indices = self._get_all_occupied_indices()
218 - for i in indices_range:
219 - if i not in all_occupied_indices:
220 - return i
221 - raise Exception("Couldn't find a free spot in the mamba cache! This"
222 - "should never happen")