A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Kernel][Model] Varlen prefill + Prefill chunking support for mamba kernels and Jamba model (#8533)
Mor Zusman committed
Sep 30, 2024 at 00:35 UTC
f13a07b1f8c11ddbdc53b40f1fbb24bf3166b900
13 files changed
+1176
-894
csrc/mamba/causal_conv1d/causal_conv1d.cu
+211
-316
@@ -39,8 +39,6 @@
39
40
template<typename input_t, typename weight_t>
41
void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream);
42
-template <typename input_t, typename weight_t>
43
-void causal_conv1d_channellast_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream);
42
43
template<typename input_t, typename weight_t>
44
void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream);
@@ -55,8 +53,11 @@ void set_conv_params_fwd(ConvParamsBase ¶ms,
53
const at::Tensor x,
54
const at::Tensor weight,
55
const at::Tensor out,
58
- void* bias_ptr,
59
- bool silu_activation) {
56
+ const c10::optional<at::Tensor>& bias,
57
+ bool silu_activation,
58
+ const c10::optional<at::Tensor>& query_start_loc = std::nullopt,
59
+ const c10::optional<at::Tensor>& cache_indices = std::nullopt,
60
+ const c10::optional<at::Tensor>& has_initial_state = std::nullopt) {
61
62
// Reset the parameters
63
memset(¶ms, 0, sizeof(params));
@@ -71,26 +72,31 @@ void set_conv_params_fwd(ConvParamsBase ¶ms,
72
// Set the pointers and strides.
73
params.x_ptr = x.data_ptr();
74
params.weight_ptr = weight.data_ptr();
74
- params.bias_ptr = bias_ptr;
75
+ params.bias_ptr = bias.has_value() ? bias.value().data_ptr() : nullptr;
76
params.out_ptr = out.data_ptr();
77
// All stride are in elements, not bytes.
77
- params.x_batch_stride = x.stride(0);
78
- params.x_c_stride = x.stride(1);
79
- params.x_l_stride = x.stride(-1);
78
+ params.query_start_loc_ptr = query_start_loc.has_value() ? query_start_loc.value().data_ptr() : nullptr;
79
+ params.cache_indices_ptr = cache_indices.has_value() ? cache_indices.value().data_ptr() : nullptr;
80
+ params.has_initial_state_ptr = has_initial_state.has_value() ? has_initial_state.value().data_ptr() : nullptr;
81
+ const bool varlen = params.query_start_loc_ptr != nullptr;
82
+ params.x_batch_stride = x.stride(varlen ? 1 : 0);
83
+ params.x_c_stride = x.stride(varlen ? 0 : 1);
84
+ params.x_l_stride = x.stride(varlen ? 1 : -1);
85
params.weight_c_stride = weight.stride(0);
86
params.weight_width_stride = weight.stride(1);
82
- params.out_batch_stride = out.stride(0);
83
- params.out_c_stride = out.stride(1);
84
- params.out_l_stride = out.stride(-1);
87
+ params.out_batch_stride = out.stride(varlen ? 1 : 0);
88
+ params.out_c_stride = out.stride(varlen ? 0 : 1);
89
+ params.out_l_stride = out.stride(varlen ? 1 : -1);
90
}
91
92
93
at::Tensor
94
causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
95
const c10::optional<at::Tensor> &bias_,
91
- const c10::optional<at::Tensor> &seq_idx_,
92
- const c10::optional<at::Tensor> &initial_states_,
93
- const c10::optional<at::Tensor> &final_states_out_,
96
+ const c10::optional<at::Tensor> &conv_states,
97
+ const c10::optional<at::Tensor> &query_start_loc,
98
+ const c10::optional<at::Tensor> &cache_indices,
99
+ const c10::optional<at::Tensor> &has_initial_state,
100
bool silu_activation) {
101
auto input_type = x.scalar_type();
102
auto weight_type = weight.scalar_type();
@@ -99,24 +105,22 @@ causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
105
106
TORCH_CHECK(x.is_cuda());
107
TORCH_CHECK(weight.is_cuda());
102
-
108
+
109
+ const bool varlen = query_start_loc.has_value() ? true : false;
110
const auto sizes = x.sizes();
104
- const int batch_size = sizes[0];
105
- const int dim = sizes[1];
106
- const int seqlen = sizes[2];
111
+ const int batch_size = varlen ? query_start_loc.value().sizes()[0] - 1 : sizes[0];
112
+ const int dim = varlen ? sizes[0] : sizes[1];
113
+ const int seqlen = varlen ? sizes[1] : sizes[2];
114
const int width = weight.size(-1);
108
-
109
- CHECK_SHAPE(x, batch_size, dim, seqlen);
115
+ if (varlen){
116
+ CHECK_SHAPE(x, dim, seqlen);
117
+ }
118
+ else {
119
+ CHECK_SHAPE(x, batch_size, dim, seqlen);
120
+ }
121
CHECK_SHAPE(weight, dim, width);
122
112
- TORCH_CHECK(x.stride(2) == 1 || x.stride(1) == 1);
113
- const bool is_channel_last = x.stride(1) == 1 && x.stride(2) > 1;
123
115
- if (is_channel_last) {
116
- TORCH_CHECK(dim % 8 == 0, "causal_conv1d only supports channel dimension divisible by 8 for now");
117
- TORCH_CHECK(x.stride(2) % 8 == 0 and x.stride(0) % 8 == 0, "causal_conv1d with channel last layout requires strides (x.stride(0) and x.stride(2)) to be multiples of 8");
118
- }
119
- TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4");
124
125
if (bias_.has_value()) {
126
auto bias = bias_.value();
@@ -126,56 +130,50 @@ causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
130
CHECK_SHAPE(bias, dim);
131
}
132
129
- if (seq_idx_.has_value()) {
130
- TORCH_CHECK(is_channel_last, "seq_idx is only supported for channel last layout");
131
- auto seq_idx = seq_idx_.value();
132
- TORCH_CHECK(seq_idx.scalar_type() == torch::kInt32);
133
- TORCH_CHECK(seq_idx.is_cuda());
134
- TORCH_CHECK(seq_idx.is_contiguous());
135
- CHECK_SHAPE(seq_idx, batch_size, seqlen);
133
+
134
+ if (has_initial_state.has_value()) {
135
+ auto has_initial_state_ = has_initial_state.value();
136
+ TORCH_CHECK(has_initial_state_.scalar_type() == at::ScalarType::Bool);
137
+ TORCH_CHECK(has_initial_state_.is_cuda());
138
+ CHECK_SHAPE(has_initial_state_, batch_size);
139
}
140
138
- at::Tensor out = torch::empty_like(x);
141
140
- ConvParamsBase params;
141
- set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
142
- bias_.has_value() ? bias_.value().data_ptr() : nullptr,
143
- silu_activation);
144
-
145
- if (seq_idx_.has_value()) {
146
- params.seq_idx_ptr = seq_idx_.value().data_ptr();
147
- } else {
148
- params.seq_idx_ptr = nullptr;
142
+ if (query_start_loc.has_value()) {
143
+ auto query_start_loc_ = query_start_loc.value();
144
+ TORCH_CHECK(query_start_loc_.scalar_type() == at::ScalarType::Int);
145
+ TORCH_CHECK(query_start_loc_.is_cuda());
146
}
147
151
- if (initial_states_.has_value()) {
152
- TORCH_CHECK(is_channel_last, "initial_states is only supported for channel last layout");
153
- auto initial_states = initial_states_.value();
154
- TORCH_CHECK(initial_states.scalar_type() == input_type);
155
- TORCH_CHECK(initial_states.is_cuda());
156
- CHECK_SHAPE(initial_states, batch_size, dim, width - 1);
157
- TORCH_CHECK(initial_states.stride(1) == 1);
158
- params.initial_states_ptr = initial_states.data_ptr();
159
- params.initial_states_batch_stride = initial_states.stride(0);
160
- params.initial_states_c_stride = initial_states.stride(1);
161
- params.initial_states_l_stride = initial_states.stride(2);
162
- } else {
163
- params.initial_states_ptr = nullptr;
148
+
149
+ if (cache_indices.has_value()) {
150
+ auto cache_indices_ = cache_indices.value();
151
+ TORCH_CHECK(cache_indices_.scalar_type() == at::ScalarType::Int);
152
+ TORCH_CHECK(cache_indices_.is_cuda());
153
+ CHECK_SHAPE(cache_indices_, batch_size);
154
}
155
166
- if (final_states_out_.has_value()) {
167
- TORCH_CHECK(is_channel_last, "final_states is only supported for channel last layout");
168
- auto final_states = final_states_out_.value();
169
- TORCH_CHECK(final_states.scalar_type() == input_type);
170
- TORCH_CHECK(final_states.is_cuda());
171
- CHECK_SHAPE(final_states, batch_size, dim, width - 1);
172
- TORCH_CHECK(final_states.stride(1) == 1);
173
- params.final_states_ptr = final_states.data_ptr();
174
- params.final_states_batch_stride = final_states.stride(0);
175
- params.final_states_c_stride = final_states.stride(1);
176
- params.final_states_l_stride = final_states.stride(2);
156
+ at::Tensor out = torch::empty_like(x);
157
+
158
+ ConvParamsBase params;
159
+ set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
160
+ bias_,
161
+ silu_activation,
162
+ query_start_loc,
163
+ cache_indices,
164
+ has_initial_state
165
+ );
166
+
167
+ if (conv_states.has_value()) {
168
+ auto conv_states_ = conv_states.value();
169
+ TORCH_CHECK(conv_states_.scalar_type() == input_type);
170
+ TORCH_CHECK(conv_states_.is_cuda());
171
+ params.conv_states_ptr = conv_states_.data_ptr();
172
+ params.conv_states_batch_stride = conv_states_.stride(0);
173
+ params.conv_states_c_stride = conv_states_.stride(1);
174
+ params.conv_states_l_stride = conv_states_.stride(2);
175
} else {
178
- params.final_states_ptr = nullptr;
176
+ params.conv_states_ptr = nullptr;
177
}
178
179
// Otherwise the kernel will be launched from cuda:0 device
@@ -183,11 +181,7 @@ causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
181
at::cuda::CUDAGuard device_guard{(char)x.get_device()};
182
auto stream = at::cuda::getCurrentCUDAStream().stream();
183
DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_fwd", [&] {
186
- if (!is_channel_last) {
187
- causal_conv1d_fwd_cuda<input_t, weight_t>(params, stream);
188
- } else {
189
- causal_conv1d_channellast_fwd_cuda<input_t, weight_t>(params, stream);
190
- }
184
+ causal_conv1d_fwd_cuda<input_t, weight_t>(params, stream);
185
});
186
return out;
187
}
@@ -199,6 +193,7 @@ causal_conv1d_update(const at::Tensor &x,
193
const at::Tensor &weight,
194
const c10::optional<at::Tensor> &bias_,
195
bool silu_activation,
196
+ const c10::optional<at::Tensor> &cache_seqlens_,
197
const c10::optional<at::Tensor> &conv_state_indices_) {
198
auto input_type = x.scalar_type();
199
auto weight_type = weight.scalar_type();
@@ -214,9 +209,12 @@ causal_conv1d_update(const at::Tensor &x,
209
const auto sizes = x.sizes();
210
const int batch_size = sizes[0];
211
const int dim = sizes[1];
212
+ const int seqlen = sizes[2];
213
const int width = weight.size(-1);
214
+ const int conv_state_len = conv_state.size(2);
215
+ TORCH_CHECK(conv_state_len >= width - 1);
216
219
- CHECK_SHAPE(x, batch_size, dim);
217
+ CHECK_SHAPE(x, batch_size, dim, seqlen);
218
CHECK_SHAPE(weight, dim, width);
219
220
TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4");
@@ -232,15 +230,27 @@ causal_conv1d_update(const at::Tensor &x,
230
at::Tensor out = torch::empty_like(x);
231
232
ConvParamsBase params;
235
- set_conv_params_fwd(params, batch_size, dim, /*seqlen=*/1, width, x, weight, out,
236
- bias_.has_value() ? bias_.value().data_ptr() : nullptr,
233
+ set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
234
+ bias_,
235
silu_activation);
236
params.conv_state_ptr = conv_state.data_ptr();
237
+ params.conv_state_len = conv_state_len;
238
// All stride are in elements, not bytes.
239
params.conv_state_batch_stride = conv_state.stride(0);
240
params.conv_state_c_stride = conv_state.stride(1);
241
params.conv_state_l_stride = conv_state.stride(2);
242
243
+ if (cache_seqlens_.has_value()) {
244
+ auto cache_seqlens = cache_seqlens_.value();
245
+ TORCH_CHECK(cache_seqlens.scalar_type() == torch::kInt32);
246
+ TORCH_CHECK(cache_seqlens.is_cuda());
247
+ TORCH_CHECK(cache_seqlens.stride(-1) == 1);
248
+ CHECK_SHAPE(cache_seqlens, batch_size);
249
+ params.cache_seqlens = cache_seqlens.data_ptr<int32_t>();
250
+ } else {
251
+ params.cache_seqlens = nullptr;
252
+ }
253
+
254
if (conv_state_indices_.has_value()) {
255
auto conv_state_indices = conv_state_indices_.value();
256
TORCH_CHECK(conv_state_indices.scalar_type() == torch::kInt32)
@@ -249,11 +259,11 @@ causal_conv1d_update(const at::Tensor &x,
259
CHECK_SHAPE(conv_state_indices, batch_size);
260
261
int conv_state_entries = conv_state.size(0);
252
- CHECK_SHAPE(conv_state, conv_state_entries, dim, width);
262
+ CHECK_SHAPE(conv_state, conv_state_entries, dim, conv_state_len);
263
264
params.conv_state_indices_ptr = conv_state_indices.data_ptr<int32_t>();
265
} else {
256
- CHECK_SHAPE(conv_state, batch_size, dim, width);
266
+ CHECK_SHAPE(conv_state, batch_size, dim, conv_state_len);
267
params.conv_state_indices_ptr = nullptr;
268
}
269
@@ -296,7 +306,7 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
306
constexpr int kWidth = Ktraits::kWidth;
307
constexpr int kNThreads = Ktraits::kNThreads;
308
constexpr int kNElts = Ktraits::kNElts;
299
- static constexpr bool kIsVecLoad = Ktraits::kIsVecLoad;
309
+ constexpr bool kIsVecLoad = Ktraits::kIsVecLoad;
310
using input_t = typename Ktraits::input_t;
311
using vec_t = typename Ktraits::vec_t;
312
using weight_t = typename Ktraits::weight_t;
@@ -309,20 +319,39 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
319
auto& smem_store_vec = reinterpret_cast<typename Ktraits::BlockStoreVecT::TempStorage&>(smem_);
320
vec_t *smem_exchange = reinterpret_cast<vec_t *>(smem_ + Ktraits::kSmemIOSize);
321
322
+ const bool kVarlen = params.query_start_loc_ptr != nullptr;
323
const int tidx = threadIdx.x;
324
const int batch_id = blockIdx.x;
325
const int channel_id = blockIdx.y;
315
- input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
326
+ const int *query_start_loc = kVarlen ? reinterpret_cast<int *>(params.query_start_loc_ptr) : nullptr;
327
+ const int sequence_start_index = kVarlen ? query_start_loc[batch_id] : batch_id;
328
+ const int seqlen = kVarlen ? query_start_loc[batch_id + 1] - sequence_start_index : params.seqlen;
329
+
330
+ input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + sequence_start_index * params.x_batch_stride
331
+ channel_id * params.x_c_stride;
332
weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr) + channel_id * params.weight_c_stride;
318
- input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
333
+ input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + sequence_start_index * params.out_batch_stride
334
+ channel_id * params.out_c_stride;
335
float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[channel_id]);
336
337
+ bool has_initial_state = params.has_initial_state_ptr == nullptr ? false
338
+ : reinterpret_cast<bool *>(params.has_initial_state_ptr)[batch_id];
339
+
340
+ int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr
341
+ : reinterpret_cast<int *>(params.cache_indices_ptr);
342
+ int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id];
343
+
344
+ input_t *conv_states = params.conv_states_ptr == nullptr ? nullptr
345
+ : reinterpret_cast<input_t *>(params.conv_states_ptr) + cache_index * params.conv_states_batch_stride + channel_id * params.conv_states_c_stride;
346
+
347
// Thread 0 will load the last elements of the previous chunk, so we initialize those to 0.
348
if (tidx == 0) {
324
- input_t zeros[kNElts] = {0};
325
- smem_exchange[kNThreads - 1] = reinterpret_cast<vec_t *>(zeros)[0];
349
+ input_t initial_state[kNElts] = {0};
350
+ if (has_initial_state) {
351
+ #pragma unroll
352
+ for (int w = 0; w < kWidth - 1; ++w){ initial_state[kNElts - 1 - (kWidth - 2) + w ] = conv_states[w]; }
353
+ }
354
+ smem_exchange[kNThreads - 1] = reinterpret_cast<vec_t *>(initial_state)[0];
355
}
356
357
float weight_vals[kWidth];
@@ -330,14 +359,14 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
359
for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); }
360
361
constexpr int kChunkSize = kNThreads * kNElts;
333
- const int n_chunks = (params.seqlen + kChunkSize - 1) / kChunkSize;
362
+ const int n_chunks = (seqlen + kChunkSize - 1) / kChunkSize;
363
for (int chunk = 0; chunk < n_chunks; ++chunk) {
364
input_t x_vals_load[2 * kNElts] = {0};
365
if constexpr(kIsVecLoad) {
337
- typename Ktraits::BlockLoadVecT(smem_load_vec).Load(reinterpret_cast<vec_t*>(x), *reinterpret_cast<vec_t (*)[1]>(&x_vals_load[kNElts]), (params.seqlen - chunk * kChunkSize) / kNElts);
366
+ typename Ktraits::BlockLoadVecT(smem_load_vec).Load(reinterpret_cast<vec_t*>(x), *reinterpret_cast<vec_t (*)[1]>(&x_vals_load[kNElts]), (seqlen - chunk * kChunkSize) / kNElts);
367
} else {
368
__syncthreads();
340
- typename Ktraits::BlockLoadT(smem_load).Load(x, *reinterpret_cast<input_t (*)[kNElts]>(&x_vals_load[kNElts]), params.seqlen - chunk * kChunkSize);
369
+ typename Ktraits::BlockLoadT(smem_load).Load(x, *reinterpret_cast<input_t (*)[kNElts]>(&x_vals_load[kNElts]), seqlen - chunk * kChunkSize);
370
}
371
x += kChunkSize;
372
__syncthreads();
@@ -375,19 +404,57 @@ void causal_conv1d_fwd_kernel(ConvParamsBase params) {
404
#pragma unroll
405
for (int i = 0; i < kNElts; ++i) { out_vals_store[i] = out_vals[i]; }
406
if constexpr(kIsVecLoad) {
378
- typename Ktraits::BlockStoreVecT(smem_store_vec).Store(reinterpret_cast<vec_t*>(out), reinterpret_cast<vec_t (&)[1]>(out_vals_store), (params.seqlen - chunk * kChunkSize) / kNElts);
407
+ typename Ktraits::BlockStoreVecT(smem_store_vec).Store(reinterpret_cast<vec_t*>(out), reinterpret_cast<vec_t (&)[1]>(out_vals_store), (seqlen - chunk * kChunkSize) / kNElts);
408
} else {
380
- typename Ktraits::BlockStoreT(smem_store).Store(out, out_vals_store, params.seqlen - chunk * kChunkSize);
409
+ typename Ktraits::BlockStoreT(smem_store).Store(out, out_vals_store, seqlen - chunk * kChunkSize);
410
}
411
out += kChunkSize;
412
}
413
+ // Final state is stored in the smem_exchange last token slot,
414
+ // in case seqlen < kWidth, we would need to take the final state from the
415
+ // initial state which is stored in conv_states
416
+ // in case seqlen > kWidth, we would need to load the last kWidth - 1 data
417
+ // and load it into conv_state accordingly
418
+ int last_thread = ((seqlen - (kWidth - 1)) - (n_chunks - 1) * kChunkSize) / kNElts;
419
+ if (conv_states != nullptr && tidx == last_thread) {
420
+ input_t x_vals_load[kNElts * 2] = {0};
421
+ // in case we are on the first kWidth tokens
422
+ if (last_thread == 0 && seqlen < kWidth){
423
+ // Need to take the initial state
424
+ reinterpret_cast<vec_t *>(x_vals_load)[0] = smem_exchange[0];
425
+ const int offset = seqlen - (kWidth - 1);
426
+ #pragma unroll
427
+ for (int w = 0; w < kWidth - 1; ++w){
428
+ // pad the existing state
429
+ if ((w - seqlen) >= 0 && has_initial_state) { conv_states[w - seqlen] = conv_states[w]; }
430
+ else if ((w - seqlen) >= 0 && !has_initial_state) { conv_states[w - seqlen] = input_t(0.0f); }
431
+ }
432
+ #pragma unroll
433
+ for (int w = 0; w < kWidth - 1; ++w){
434
+ if (offset + w >= 0)
435
+ conv_states[w] = x_vals_load[offset + w ];
436
+ }
437
+ }
438
+ else {
439
+ // in case the final state is in between the threads data
440
+ reinterpret_cast<vec_t *>(x_vals_load)[1] = smem_exchange[last_thread + 1];
441
+ reinterpret_cast<vec_t *>(x_vals_load)[0] = smem_exchange[last_thread];
442
+ const int offset = ((seqlen - (kWidth - 1)) % (kNElts));
443
+ #pragma unroll
444
+ for (int w = 0; w < kWidth - 1; ++w){
445
+ conv_states[w] = x_vals_load[offset + w ];
446
+ }
447
+ }
448
+
449
+ }
450
}
451
452
453
template<int kNThreads, int kWidth, typename input_t, typename weight_t>
454
void causal_conv1d_fwd_launch(ConvParamsBase ¶ms, cudaStream_t stream) {
455
static constexpr int kNElts = sizeof(input_t) == 4 ? 4 : 8;
390
- BOOL_SWITCH(params.seqlen % kNElts == 0, kIsVecLoad, [&] {
456
+ const bool kVarlen = params.query_start_loc_ptr != nullptr;
457
+ BOOL_SWITCH(params.seqlen % kNElts == 0 && !kVarlen, kIsVecLoad, [&] {
458
using Ktraits = Causal_conv1d_fwd_kernel_traits<kNThreads, kWidth, kIsVecLoad, input_t, weight_t>;
459
constexpr int kSmemSize = Ktraits::kSmemSize;
460
dim3 grid(params.batch, params.dim);
@@ -422,220 +489,11 @@ void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream) {
489
}
490
}
491
425
-template<int kNThreads_, int kWidth_, int kChunkSizeL_, bool kIsVecLoad_, typename input_t_, typename weight_t_>
426
-struct Causal_conv1d_channellast_fwd_kernel_traits {
427
- // The cache line is 128 bytes, and we try to read 16 bytes per thread.
428
- // So we have 8 threads per "row", so 32 or 64 elements in the channel dimension.
429
- // That leaves 4 columns per warp, and so 16 columns per block (assuming each block has 128
430
- // threads). Each each load is 16 x 32|64 elements in the L x C dimensions.
431
- using input_t = input_t_;
432
- using weight_t = weight_t_;
433
- static constexpr int kNThreads = kNThreads_;
434
- static_assert(kNThreads % 32 == 0);
435
- static constexpr int kNWarps = kNThreads / 32;
436
- static constexpr int kWidth = kWidth_;
437
- static constexpr int kChunkSizeL = kChunkSizeL_;
438
- static constexpr int kNBytes = sizeof(input_t);
439
- static_assert(kNBytes == 2 || kNBytes == 4);
440
- static constexpr int kNElts = kNBytes == 4 ? 4 : 8;
441
- static constexpr int kNEltsPerRow = 128 / kNBytes;
442
- static constexpr int kNThreadsPerRow = kNEltsPerRow / kNElts; // Always 8 for now
443
- static_assert(kNThreadsPerRow * kNBytes * kNElts == 128);
444
- static constexpr int kNColsPerWarp = 32 / kNThreadsPerRow; // Always 4 for now
445
- static_assert(kNColsPerWarp * kNThreadsPerRow == 32);
446
- static constexpr int kNColsPerLoad = kNColsPerWarp * kNWarps;
447
- static constexpr int kNLoads = kChunkSizeL / kNColsPerLoad;
448
- static_assert(kNLoads * kNColsPerLoad == kChunkSizeL);
449
- static constexpr bool kIsVecLoad = kIsVecLoad_;
450
- using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
451
- // using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
452
- // using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNItems, cub::BLOCK_STORE_WARP_TRANSPOSE>;
453
- // static constexpr int kSmemSize = std::max({sizeof(typename BlockLoadT::TempStorage),
454
- // sizeof(typename BlockStoreT::TempStorage)});
455
- // static constexpr int kSmemSize = kChunkSizeL * kNEltsPerRow * kNBytes;
456
-};
457
-
458
-template<typename Ktraits, bool kHasSeqIdx>
459
-__global__ __launch_bounds__(Ktraits::kNThreads)
460
-void causal_conv1d_channellast_fwd_kernel(ConvParamsBase params) {
461
- constexpr int kWidth = Ktraits::kWidth;
462
- constexpr int kNThreads = Ktraits::kNThreads;
463
- constexpr int kNElts = Ktraits::kNElts;
464
- constexpr int kNThreadsPerC = Ktraits::kNThreadsPerRow;
465
- constexpr int kLPerLoad = Ktraits::kNColsPerLoad;
466
- constexpr int kChunkSizeL = Ktraits::kChunkSizeL;
467
- constexpr int kChunkSizeC = Ktraits::kNEltsPerRow;
468
- using input_t = typename Ktraits::input_t;
469
- using vec_t = typename Ktraits::vec_t;
470
- using weight_t = typename Ktraits::weight_t;
471
-
472
- // Shared memory.
473
- __shared__ input_t x_smem[kWidth - 1 + kChunkSizeL][kChunkSizeC + kNElts];
474
-
475
- const int batch_id = blockIdx.x;
476
- const int chunk_l_id = blockIdx.y;
477
- const int chunk_c_id = blockIdx.z;
478
- const int tid = threadIdx.x;
479
- const int l_idx = tid / kNThreadsPerC;
480
- const int c_idx = tid % kNThreadsPerC;
481
- input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
482
- + (chunk_l_id * kChunkSizeL + l_idx) * params.x_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
483
- weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr)
484
- + chunk_c_id * kChunkSizeC * params.weight_c_stride;
485
- input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
486
- + (chunk_l_id * kChunkSizeL + l_idx) * params.out_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
487
- int *seq_idx = !kHasSeqIdx ? nullptr : reinterpret_cast<int *>(params.seq_idx_ptr)
488
- + batch_id * params.seqlen + chunk_l_id * kChunkSizeL;
489
- input_t *initial_states = params.initial_states_ptr == nullptr || chunk_l_id > 0 ? nullptr
490
- : reinterpret_cast<input_t *>(params.initial_states_ptr) + batch_id * params.initial_states_batch_stride + l_idx * params.initial_states_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
491
- // The last L-chunk will also have enough info to write to final states, since it also contain a few x values
492
- // from the previous L-chunk.
493
- input_t *final_states = params.final_states_ptr == nullptr || chunk_l_id < gridDim.y - 1 ? nullptr
494
- : reinterpret_cast<input_t *>(params.final_states_ptr) + batch_id * params.final_states_batch_stride + l_idx * params.final_states_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
495
-
496
- #pragma unroll
497
- for (int l = 0; l < Ktraits::kNLoads; ++l) {
498
- input_t x_vals_load[kNElts] = {0};
499
- if (chunk_l_id * kChunkSizeL + l * kLPerLoad + l_idx < params.seqlen
500
- && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
501
- reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(x + l * kLPerLoad * params.x_l_stride);
502
- }
503
- reinterpret_cast<vec_t *>(x_smem[kWidth - 1 + l * kLPerLoad + l_idx])[c_idx] = reinterpret_cast<vec_t *>(x_vals_load)[0];
504
- }
505
- // Load the elements from the previous chunk that are needed for convolution.
506
- if (l_idx < kWidth - 1) {
507
- input_t x_vals_load[kNElts] = {0};
508
- if (chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) >= 0
509
- && chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) < params.seqlen
510
- && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
511
- reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(x - (kWidth - 1) * params.x_l_stride);
512
- } else if (initial_states != nullptr
513
- && chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) < 0
514
- && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
515
- reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(initial_states);
516
- }
517
- reinterpret_cast<vec_t *>(x_smem[l_idx])[c_idx] = reinterpret_cast<vec_t *>(x_vals_load)[0];
518
- }
519
-
520
- __syncthreads();
521
-
522
- if (final_states != nullptr
523
- && l_idx < kWidth - 1
524
- && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
525
- // x_smem[0] contains element at index chunk_l_id * kChunkSizeL - (kWidth - 1)
526
- // So last few elements (index params.seqlen - kWidth + 1 + l_idx) are stored in x_smem[params.seqlen - kWidth + 1 + l_idx - (chunk_l_id * kChunkSizeL - kWidth + 1)][c_idx]
527
- *reinterpret_cast<vec_t *>(final_states) = reinterpret_cast<vec_t *>(x_smem[params.seqlen + l_idx - chunk_l_id * kChunkSizeL])[c_idx];
528
- }
529
-
530
- constexpr int kLPerThread = constexpr_min(kChunkSizeL * kChunkSizeC / kNThreads, kChunkSizeL);
531
- static_assert(kLPerThread * kNThreads == kChunkSizeL * kChunkSizeC);
532
- constexpr int kNThreadsPerRow = kChunkSizeL / kLPerThread;
533
- static_assert(kNThreadsPerRow * kLPerThread == kChunkSizeL);
534
- // kChunkSizeL, kLPerThread, kNThreadsPerRow should be powers of 2 for simplicity
535
- static_assert((kChunkSizeL & (kChunkSizeL - 1)) == 0);
536
- static_assert((kLPerThread & (kLPerThread - 1)) == 0);
537
- static_assert((kNThreadsPerRow & (kNThreadsPerRow - 1)) == 0);
538
- static_assert(kNThreadsPerRow <= 32);
539
-
540
- const int row_idx = tid / kNThreadsPerRow;
541
- const int col_idx = tid % kNThreadsPerRow;
542
-
543
- float bias_val = params.bias_ptr == nullptr || chunk_c_id * kChunkSizeC + row_idx >= params.dim ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[chunk_c_id * kChunkSizeC + row_idx]);
544
- float weight_vals[kWidth] = {0};
545
- if (chunk_c_id * kChunkSizeC + row_idx < params.dim) {
546
- #pragma unroll
547
- for (int w = 0; w < kWidth; ++w) {
548
- weight_vals[w] = weight[row_idx * params.weight_c_stride + w * params.weight_width_stride];
549
- }
550
- }
551
- float x_vals[kWidth - 1 + kLPerThread];
552
- #pragma unroll
553
- for (int i = 0; i < kWidth - 1 + kLPerThread; ++i) {
554
- x_vals[i] = float(x_smem[col_idx * kLPerThread + i][row_idx]);
555
- }
556
- int seq_idx_thread[kWidth - 1 + kLPerThread];
557
- if constexpr (kHasSeqIdx) {
558
- #pragma unroll
559
- for (int i = 0; i < kWidth - 1 + kLPerThread; ++i) {
560
- seq_idx_thread[i] = chunk_l_id * kChunkSizeL + col_idx * kLPerThread + i - (kWidth - 1) >= 0 ? seq_idx[col_idx * kLPerThread + i - (kWidth - 1)] : -1;
561
- }
562
- }
563
-
564
- float out_vals[kLPerThread];
565
- #pragma unroll
566
- for (int i = 0; i < kLPerThread; ++i) {
567
- out_vals[i] = bias_val;
568
- const int seq_idx_cur = !kHasSeqIdx ? 0 : seq_idx_thread[i + kWidth - 1];
569
- #pragma unroll
570
- for (int w = 0; w < kWidth; ++w) {
571
- if constexpr (!kHasSeqIdx) {
572
- out_vals[i] += weight_vals[w] * x_vals[i + w];
573
- } else {
574
- out_vals[i] += seq_idx_thread[i + w] == seq_idx_cur ? weight_vals[w] * x_vals[i + w] : 0.f;
575
- }
576
- }
577
- if (params.silu_activation) {out_vals[i] = out_vals[i] / (1 + expf(-out_vals[i])); }
578
- }
579
-
580
- __syncthreads();
581
- #pragma unroll
582
- for (int i = 0; i < kLPerThread; ++i) { x_smem[col_idx * kLPerThread + i][row_idx] = out_vals[i]; }
583
- __syncthreads();
584
-
585
- #pragma unroll
586
- for (int l = 0; l < Ktraits::kNLoads; ++l) {
587
- input_t out_vals_store[kNElts];
588
- reinterpret_cast<vec_t *>(out_vals_store)[0] = reinterpret_cast<vec_t *>(x_smem[l * kLPerLoad + l_idx])[c_idx];
589
- if (chunk_l_id * kChunkSizeL + l * kLPerLoad + l_idx < params.seqlen
590
- && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
591
- *reinterpret_cast<vec_t *>(out + l * kLPerLoad * params.out_l_stride) = reinterpret_cast<vec_t *>(out_vals_store)[0];
592
- }
593
- }
594
-
595
-}
596
-
597
-template<int kNThreads, int kWidth, typename input_t, typename weight_t>
598
-void causal_conv1d_channellast_fwd_launch(ConvParamsBase ¶ms, cudaStream_t stream) {
599
- BOOL_SWITCH(params.seq_idx_ptr != nullptr, kHasSeqIdx, [&] {
600
- using Ktraits = Causal_conv1d_channellast_fwd_kernel_traits<kNThreads, kWidth, 64, true, input_t, weight_t>;
601
- // constexpr int kSmemSize = Ktraits::kSmemSize;
602
- constexpr int kChunkSizeL = Ktraits::kChunkSizeL;
603
- constexpr int kChunkSizeC = Ktraits::kNEltsPerRow;
604
- const int n_chunks_L = (params.seqlen + kChunkSizeL - 1) / kChunkSizeL;
605
- const int n_chunks_C = (params.dim + kChunkSizeC - 1) / kChunkSizeC;
606
- dim3 grid(params.batch, n_chunks_L, n_chunks_C);
607
- dim3 block(Ktraits::kNThreads);
608
- auto kernel = &causal_conv1d_channellast_fwd_kernel<Ktraits, kHasSeqIdx>;
609
- // if (kSmemSize >= 48 * 1024) {
610
- // C10_CUDA_CHECK(cudaFuncSetAttribute(
611
- // kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
612
- // }
613
- // kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
614
- kernel<<<grid, Ktraits::kNThreads, 0, stream>>>(params);
615
- C10_CUDA_KERNEL_LAUNCH_CHECK();
616
- });
617
-}
618
-
619
-template<typename input_t, typename weight_t>
620
-void causal_conv1d_channellast_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream) {
621
- if (params.width == 2) {
622
- causal_conv1d_channellast_fwd_launch<128, 2, input_t, weight_t>(params, stream);
623
- } else if (params.width == 3) {
624
- causal_conv1d_channellast_fwd_launch<128, 3, input_t, weight_t>(params, stream);
625
- } else if (params.width == 4) {
626
- causal_conv1d_channellast_fwd_launch<128, 4, input_t, weight_t>(params, stream);
627
- }
628
-}
492
493
template void causal_conv1d_fwd_cuda<float, float>(ConvParamsBase ¶ms, cudaStream_t stream);
494
template void causal_conv1d_fwd_cuda<at::Half, at::Half>(ConvParamsBase ¶ms, cudaStream_t stream);
495
template void causal_conv1d_fwd_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase ¶ms, cudaStream_t stream);
496
634
-template void causal_conv1d_channellast_fwd_cuda<float, float>(ConvParamsBase ¶ms, cudaStream_t stream);
635
-template void causal_conv1d_channellast_fwd_cuda<at::Half, at::Half>(ConvParamsBase ¶ms, cudaStream_t stream);
636
-template void causal_conv1d_channellast_fwd_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase ¶ms, cudaStream_t stream);
637
-///////
638
-
497
498
499
@@ -649,7 +507,7 @@ struct Causal_conv1d_update_kernel_traits {
507
static_assert(kNBytes == 2 || kNBytes == 4);
508
};
509
652
-template<typename Ktraits>
510
+template<typename Ktraits, bool kIsCircularBuffer>
511
__global__ __launch_bounds__(Ktraits::kNThreads)
512
void causal_conv1d_update_kernel(ConvParamsBase params) {
513
constexpr int kWidth = Ktraits::kWidth;
@@ -660,6 +518,8 @@ void causal_conv1d_update_kernel(ConvParamsBase params) {
518
const int tidx = threadIdx.x;
519
const int batch_id = blockIdx.x;
520
const int channel_id = blockIdx.y * kNThreads + tidx;
521
+ if (channel_id >= params.dim) return;
522
+
523
input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
524
+ channel_id * params.x_c_stride;
525
@@ -675,35 +535,70 @@ void causal_conv1d_update_kernel(ConvParamsBase params) {
535
weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr) + channel_id * params.weight_c_stride;
536
input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
537
+ channel_id * params.out_c_stride;
678
- float bias_val = params.bias_ptr == nullptr || channel_id >= params.dim ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[channel_id]);
538
+ float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[channel_id]);
539
+
540
+ int state_len = params.conv_state_len;
541
+ int advance_len = params.seqlen;
542
+ int cache_seqlen = kIsCircularBuffer ? params.cache_seqlens[batch_id] % state_len : 0;
543
+ int update_idx = cache_seqlen - (kWidth - 1);
544
+ update_idx = update_idx < 0 ? update_idx + state_len : update_idx;
545
546
float weight_vals[kWidth] = {0};
681
- if (channel_id < params.dim) {
682
- #pragma unroll
683
- for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); }
684
- }
547
+ #pragma unroll
548
+ for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); }
549
550
float x_vals[kWidth] = {0};
687
- if (channel_id < params.dim) {
551
+ if constexpr (!kIsCircularBuffer) {
552
+ #pragma unroll 2
553
+ for (int i = 0; i < state_len - advance_len - (kWidth - 1); ++i) {
554
+ conv_state[i * params.conv_state_l_stride] = conv_state[(i + advance_len) * params.conv_state_l_stride];
555
+ }
556
#pragma unroll
689
- for (int i = 0; i < kWidth - 1; ++i) { x_vals[i] = float(conv_state[(i + 1) * params.conv_state_l_stride]); }
690
- x_vals[kWidth - 1] = float(x[0]);
557
+ for (int i = 0; i < kWidth - 1; ++i) {
558
+ input_t state_val = conv_state[(state_len - (kWidth - 1) + i) * params.conv_state_l_stride];
559
+ if (i < advance_len + (kWidth - 1) && state_len - advance_len - (kWidth - 1) + i >= 0) {
560
+ conv_state[(state_len - advance_len - (kWidth - 1) + i) * params.conv_state_l_stride] = state_val;
561
+ }
562
+ x_vals[i] = float(state_val);
563
+ }
564
+ } else {
565
#pragma unroll
692
- for (int i = 0; i < kWidth; ++i) { conv_state[i * params.conv_state_l_stride] = input_t(x_vals[i]); }
566
+ for (int i = 0; i < kWidth - 1; ++i, update_idx = update_idx + 1 >= state_len ? update_idx + 1 - state_len : update_idx + 1) {
567
+ input_t state_val = conv_state[update_idx * params.conv_state_l_stride];
568
+ x_vals[i] = float(state_val);
569
+ }
570
+ }
571
+ #pragma unroll 2
572
+ for (int i = 0; i < params.seqlen; ++i) {
573
+ input_t x_val = x[i * params.x_l_stride];
574
+ if constexpr (!kIsCircularBuffer) {
575
+ if (i < advance_len && state_len - advance_len + i >= 0) {
576
+ conv_state[(state_len - advance_len + i) * params.conv_state_l_stride] = x_val;
577
+ }
578
+ } else {
579
+ conv_state[update_idx * params.conv_state_l_stride] = x_val;
580
+ ++update_idx;
581
+ update_idx = update_idx >= state_len ? update_idx - state_len : update_idx;
582
+ }
583
+ x_vals[kWidth - 1] = float(x_val);
584
+ float out_val = bias_val;
585
+ #pragma unroll
586
+ for (int j = 0; j < kWidth; ++j) { out_val += weight_vals[j] * x_vals[j]; }
587
+ if (params.silu_activation) { out_val = out_val / (1 + expf(-out_val)); }
588
+ out[i * params.out_l_stride] = input_t(out_val);
589
+ // Shift the input buffer by 1
590
+ #pragma unroll
591
+ for (int i = 0; i < kWidth - 1; ++i) { x_vals[i] = x_vals[i + 1]; }
592
}
694
-
695
- float out_val = bias_val;
696
- #pragma unroll
697
- for (int i = 0; i < kWidth; ++i) { out_val += weight_vals[i] * x_vals[i]; }
698
- if (params.silu_activation) { out_val = out_val / (1 + expf(-out_val)); }
699
- if (channel_id < params.dim) { out[0] = input_t(out_val); }
593
}
594
595
template<int kNThreads, int kWidth, typename input_t, typename weight_t>
596
void causal_conv1d_update_launch(ConvParamsBase ¶ms, cudaStream_t stream) {
597
using Ktraits = Causal_conv1d_update_kernel_traits<kNThreads, kWidth, input_t, weight_t>;
598
dim3 grid(params.batch, (params.dim + kNThreads - 1) / kNThreads);
706
- auto kernel = &causal_conv1d_update_kernel<Ktraits>;
599
+ auto kernel = params.cache_seqlens == nullptr
600
+ ? &causal_conv1d_update_kernel<Ktraits, false>
601
+ : &causal_conv1d_update_kernel<Ktraits, true>;
602
kernel<<<grid, Ktraits::kNThreads, 0, stream>>>(params);
603
C10_CUDA_KERNEL_LAUNCH_CHECK();
604
}
csrc/mamba/causal_conv1d/causal_conv1d.h
+10
@@ -24,6 +24,7 @@ struct ConvParamsBase {
24
index_t out_c_stride;
25
index_t out_l_stride;
26
27
+ int conv_state_len;
28
index_t conv_state_batch_stride;
29
index_t conv_state_c_stride;
30
index_t conv_state_l_stride;
@@ -35,6 +36,10 @@ struct ConvParamsBase {
36
void *__restrict__ out_ptr;
37
38
void *__restrict__ conv_state_ptr;
39
+ void *__restrict__ query_start_loc_ptr;
40
+ void *__restrict__ has_initial_state_ptr;
41
+ void *__restrict__ cache_indices_ptr;
42
+ int32_t *__restrict__ cache_seqlens;
43
44
// For the continuous batching case. Makes it so that the mamba state for
45
// the current batch doesn't need to be a contiguous tensor.
@@ -52,6 +57,11 @@ struct ConvParamsBase {
57
index_t final_states_batch_stride;
58
index_t final_states_l_stride;
59
index_t final_states_c_stride;
60
+
61
+ void * conv_states_ptr;
62
+ index_t conv_states_batch_stride;
63
+ index_t conv_states_l_stride;
64
+ index_t conv_states_c_stride;
65
};
66
67
csrc/mamba/mamba_ssm/selective_scan.h
+9
-20
@@ -54,10 +54,14 @@ struct SSMParamsBase {
54
void *__restrict__ delta_ptr;
55
void *__restrict__ delta_bias_ptr;
56
void *__restrict__ out_ptr;
57
- void *__restrict__ x_ptr;
57
+ void *__restrict__ ssm_states_ptr;
58
void *__restrict__ z_ptr;
59
void *__restrict__ out_z_ptr;
60
- void *__restrict__ index_ptr;
60
+
61
+ void *__restrict__ query_start_loc_ptr;
62
+ void *__restrict__ cache_indices_ptr;
63
+ void *__restrict__ has_initial_state_ptr;
64
+
65
};
66
67
@@ -201,7 +205,7 @@ inline __device__ void load_input(typename Ktraits::input_t *u,
205
typename Ktraits::input_t (&u_vals)[Ktraits::kNItems],
206
typename Ktraits::BlockLoadT::TempStorage &smem_load,
207
int seqlen) {
204
- if constexpr (Ktraits::kIsEvenLen) {
208
+ if constexpr (Ktraits::kIsEvenLen && !Ktraits::kVarlen) {
209
auto& smem_load_vec = reinterpret_cast<typename Ktraits::BlockLoadVecT::TempStorage&>(smem_load);
210
using vec_t = typename Ktraits::vec_t;
211
typename Ktraits::BlockLoadVecT(smem_load_vec).Load(
@@ -217,21 +221,6 @@ inline __device__ void load_input(typename Ktraits::input_t *u,
221
}
222
}
223
220
-template<typename Ktraits>
221
-inline __device__ void load_index(int *u,
222
- int (&u_vals)[Ktraits::kNItems],
223
- typename Ktraits::BlockLoadIndexT::TempStorage &smem_load_index,
224
- int seqlen) {
225
- if constexpr (Ktraits::kIsEvenLen) {
226
- auto& smem_load_index_vec = reinterpret_cast<typename Ktraits::BlockLoadIndexVecT::TempStorage&>(smem_load_index);
227
- Ktraits::BlockLoadIndexVecT(smem_load_index_vec).Load(
228
- reinterpret_cast<uint4*>(u),
229
- reinterpret_cast<uint4(&)[Ktraits::kNLoadsIndex]>(u_vals)
230
- );
231
- } else {
232
- Ktraits::BlockLoadIndexT(smem_load_index).Load(u, u_vals, seqlen, 0);
233
- }
234
-}
224
225
template<typename Ktraits>
226
inline __device__ void load_weight(typename Ktraits::input_t *Bvar,
@@ -240,7 +229,7 @@ inline __device__ void load_weight(typename Ktraits::input_t *Bvar,
229
int seqlen) {
230
constexpr int kNItems = Ktraits::kNItems;
231
typename Ktraits::input_t B_vals_load[kNItems];
243
- if constexpr (Ktraits::kIsEvenLen) {
232
+ if constexpr (Ktraits::kIsEvenLen && !Ktraits::kVarlen) {
233
auto& smem_load_weight_vec = reinterpret_cast<typename Ktraits::BlockLoadWeightVecT::TempStorage&>(smem_load_weight);
234
using vec_t = typename Ktraits::vec_t;
235
typename Ktraits::BlockLoadWeightVecT(smem_load_weight_vec).Load(
@@ -263,7 +252,7 @@ inline __device__ void store_output(typename Ktraits::input_t *out,
252
typename Ktraits::input_t write_vals[Ktraits::kNItems];
253
#pragma unroll
254
for (int i = 0; i < Ktraits::kNItems; ++i) { write_vals[i] = out_vals[i]; }
266
- if constexpr (Ktraits::kIsEvenLen) {
255
+ if constexpr (Ktraits::kIsEvenLen && !Ktraits::kVarlen) {
256
auto& smem_store_vec = reinterpret_cast<typename Ktraits::BlockStoreVecT::TempStorage&>(smem_store);
257
using vec_t = typename Ktraits::vec_t;
258
typename Ktraits::BlockStoreVecT(smem_store_vec).Store(
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
+179
-118
@@ -23,7 +23,7 @@
23
24
template<int kNThreads_, int kNItems_, int kNRows_, bool kIsEvenLen_,
25
bool kIsVariableB_, bool kIsVariableC_,
26
- bool kHasZ_, bool kUseIndex_, typename input_t_, typename weight_t_>
26
+ bool kHasZ_, bool kVarlen_, typename input_t_, typename weight_t_>
27
struct Selective_Scan_fwd_kernel_traits {
28
static_assert(kNItems_ % 4 == 0);
29
using input_t = input_t_;
@@ -38,22 +38,19 @@ struct Selective_Scan_fwd_kernel_traits {
38
static constexpr int kNElts = kNBytes == 4 ? 4 : constexpr_min(8, kNItems);
39
static_assert(kNItems % kNElts == 0);
40
static constexpr int kNLoads = kNItems / kNElts;
41
- static constexpr bool kIsEvenLen = kIsEvenLen_;
41
+ static constexpr bool kIsEvenLen = kVarlen_ ? false : kIsEvenLen_;
42
static constexpr bool kIsVariableB = kIsVariableB_;
43
static constexpr bool kIsVariableC = kIsVariableC_;
44
static constexpr bool kHasZ = kHasZ_;
45
- static constexpr bool kUseIndex = kUseIndex_;
45
+ static constexpr bool kVarlen = kVarlen_;
46
47
- static constexpr bool kDirectIO = kIsEvenLen && kNLoads == 1;
47
+ static constexpr bool kDirectIO = kVarlen_ ? false : kIsEvenLen && kNLoads == 1;
48
static constexpr int kNLoadsIndex = kNItems / 4;
49
using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
50
using scan_t = float2;
51
using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
52
using BlockLoadVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads,
53
!kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
54
- using BlockLoadIndexT = cub::BlockLoad<int, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
55
- using BlockLoadIndexVecT = cub::BlockLoad<uint4, kNThreads, kNLoadsIndex,
56
- !(kIsEvenLen && kNLoadsIndex == 1) ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
54
using BlockLoadWeightT = cub::BlockLoad<input_t, kNThreads, kNItems , cub::BLOCK_LOAD_WARP_TRANSPOSE>;
55
using BlockLoadWeightVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads ,
56
!kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
@@ -65,8 +62,6 @@ struct Selective_Scan_fwd_kernel_traits {
62
using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_WARP_SCANS>;
63
static constexpr int kSmemIOSize = custom_max({sizeof(typename BlockLoadT::TempStorage),
64
sizeof(typename BlockLoadVecT::TempStorage),
68
- sizeof(typename BlockLoadIndexT::TempStorage),
69
- sizeof(typename BlockLoadIndexVecT::TempStorage),
65
(int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightT::TempStorage),
66
(int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightVecT::TempStorage),
67
sizeof(typename BlockStoreT::TempStorage),
@@ -80,7 +75,7 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
75
constexpr bool kIsVariableB = Ktraits::kIsVariableB;
76
constexpr bool kIsVariableC = Ktraits::kIsVariableC;
77
constexpr bool kHasZ = Ktraits::kHasZ;
83
- constexpr bool kUseIndex = Ktraits::kUseIndex;
78
+ constexpr bool kVarlen = Ktraits::kVarlen;
79
constexpr int kNThreads = Ktraits::kNThreads;
80
constexpr int kNItems = Ktraits::kNItems;
81
constexpr int kNRows = Ktraits::kNRows;
@@ -97,7 +92,6 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
92
// auto& smem_load = reinterpret_cast<typename BlockLoadT::TempStorage&>(smem_loadstorescan);
93
auto& smem_load = reinterpret_cast<typename Ktraits::BlockLoadT::TempStorage&>(smem_);
94
auto& smem_load_weight = reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage&>(smem_);
100
- auto& smem_load_index = reinterpret_cast<typename Ktraits::BlockLoadIndexT::TempStorage&>(smem_);
95
auto& smem_load_weight1 = *reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage*>(smem_ + sizeof(typename Ktraits::BlockLoadWeightT::TempStorage));
96
auto& smem_store = reinterpret_cast<typename Ktraits::BlockStoreT::TempStorage&>(smem_);
97
auto& smem_scan = *reinterpret_cast<typename Ktraits::BlockScanT::TempStorage*>(smem_ + Ktraits::kSmemIOSize);
@@ -108,17 +102,29 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
102
const int batch_id = blockIdx.x;
103
const int dim_id = blockIdx.y;
104
const int group_id = dim_id / (params.dim_ngroups_ratio);
111
- input_t *u = reinterpret_cast<input_t *>(params.u_ptr) + batch_id * params.u_batch_stride
105
+ int seqlen = params.seqlen;
106
+ int sequence_start_index = batch_id;
107
+ if constexpr (kVarlen){
108
+ int *query_start_loc = reinterpret_cast<int *>(params.query_start_loc_ptr);
109
+ sequence_start_index = query_start_loc[batch_id];
110
+ seqlen = query_start_loc[batch_id + 1] - sequence_start_index;
111
+ }
112
+ const bool has_initial_state = params.has_initial_state_ptr == nullptr ? false
113
+ : reinterpret_cast<bool *>(params.has_initial_state_ptr)[batch_id];
114
+
115
+ const int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr
116
+ : reinterpret_cast<int *>(params.cache_indices_ptr);
117
+ const int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id];
118
+ input_t *u = reinterpret_cast<input_t *>(params.u_ptr) + sequence_start_index * params.u_batch_stride
119
+ dim_id * kNRows * params.u_d_stride;
113
- input_t *delta = reinterpret_cast<input_t *>(params.delta_ptr) + batch_id * params.delta_batch_stride
120
+ input_t *delta = reinterpret_cast<input_t *>(params.delta_ptr) + sequence_start_index * params.delta_batch_stride
121
+ dim_id * kNRows * params.delta_d_stride;
122
weight_t *A = reinterpret_cast<weight_t *>(params.A_ptr) + dim_id * kNRows * params.A_d_stride;
123
weight_t *B = reinterpret_cast<weight_t *>(params.B_ptr) + dim_id * kNRows * params.B_d_stride;
117
- input_t *Bvar = reinterpret_cast<input_t *>(params.B_ptr) + batch_id * params.B_batch_stride + group_id * params.B_group_stride;
124
+ input_t *Bvar = reinterpret_cast<input_t *>(params.B_ptr) + sequence_start_index * params.B_batch_stride + group_id * params.B_group_stride;
125
weight_t *C = reinterpret_cast<weight_t *>(params.C_ptr) + dim_id * kNRows * params.C_d_stride;
119
- input_t *Cvar = reinterpret_cast<input_t *>(params.C_ptr) + batch_id * params.C_batch_stride + group_id * params.C_group_stride;
120
- scan_t *x = reinterpret_cast<scan_t *>(params.x_ptr) + (batch_id * params.dim + dim_id * kNRows) * params.n_chunks * params.dstate;
121
- int *index = !kUseIndex ? nullptr :reinterpret_cast<int *>(params.index_ptr) + batch_id * params.seqlen;
126
+ input_t *Cvar = reinterpret_cast<input_t *>(params.C_ptr) + sequence_start_index * params.C_batch_stride + group_id * params.C_group_stride;
127
+ input_t *ssm_states = reinterpret_cast<input_t *>(params.ssm_states_ptr) + (cache_index * params.dim + dim_id * kNRows) * params.dstate;
128
129
float D_val[kNRows] = {0};
130
if (params.D_ptr != nullptr) {
@@ -142,9 +148,9 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
148
// }
149
150
constexpr int kChunkSize = kNThreads * kNItems;
145
- for (int chunk = 0; chunk < params.n_chunks; ++chunk) {
151
+ const int n_chunks = (seqlen + 2048 - 1) / 2048;
152
+ for (int chunk = 0; chunk < n_chunks; ++chunk) {
153
input_t u_vals[kNRows][kNItems], delta_vals_load[kNRows][kNItems];
147
- int index_vals_load[kNRows][kNItems];
154
155
__syncthreads();
156
#pragma unroll
@@ -152,15 +158,9 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
158
if constexpr (!kDirectIO) {
159
if (r > 0) { __syncthreads(); }
160
}
155
- load_input<Ktraits>(u + r * params.u_d_stride, u_vals[r], smem_load, params.seqlen - chunk * kChunkSize);
161
+ load_input<Ktraits>(u + r * params.u_d_stride, u_vals[r], smem_load, seqlen - chunk * kChunkSize);
162
if constexpr (!kDirectIO) { __syncthreads(); }
157
- load_input<Ktraits>(delta + r * params.delta_d_stride, delta_vals_load[r], smem_load, params.seqlen - chunk * kChunkSize);
158
- if constexpr (kUseIndex) {
159
- load_index<Ktraits>(index + r * params.delta_d_stride, index_vals_load[r], smem_load_index, params.seqlen - chunk * kChunkSize);
160
- }
161
- }
162
- if constexpr (kUseIndex) {
163
- index += kChunkSize;
163
+ load_input<Ktraits>(delta + r * params.delta_d_stride, delta_vals_load[r], smem_load, seqlen - chunk * kChunkSize);
164
}
165
u += kChunkSize;
166
delta += kChunkSize;
@@ -195,9 +195,9 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
195
// If both B and C vary, this is unused.
196
weight_t BC_val[kNRows];
197
weight_t B_vals[kNItems], C_vals[kNItems];
198
- if constexpr (kIsVariableB) {
198
+ if constexpr (kIsVariableB) {
199
load_weight<Ktraits>(Bvar + state_idx * params.B_dstate_stride, B_vals,
200
- smem_load_weight, (params.seqlen - chunk * kChunkSize) * (1));
200
+ smem_load_weight, (seqlen - chunk * kChunkSize) * (1));
201
if constexpr (!kIsVariableC) {
202
#pragma unroll
203
for (int r = 0; r < kNRows; ++r) {
@@ -208,7 +208,7 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
208
if constexpr (kIsVariableC) {
209
auto &smem_load_weight_C = !kIsVariableB ? smem_load_weight : smem_load_weight1;
210
load_weight<Ktraits>(Cvar + state_idx * params.C_dstate_stride, C_vals,
211
- smem_load_weight_C, (params.seqlen - chunk * kChunkSize) * (1 ));
211
+ smem_load_weight_C, (seqlen - chunk * kChunkSize) * (1 ));
212
if constexpr (!kIsVariableB) {
213
#pragma unroll
214
for (int r = 0; r < kNRows; ++r) {
@@ -232,24 +232,16 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
232
thread_data[i] = make_float2(exp2f(delta_vals[r][i] * A_val[r]),
233
!kIsVariableB ? delta_u_vals[r][i] : B_vals[i] * delta_u_vals[r][i]);
234
235
- // Reset A bar for cumulative sequences (Real)
236
- if constexpr (kUseIndex) {
237
- if (index_vals_load[r][i] == 0) {
238
- thread_data[i].x = 0.f;
239
- }
240
- }
241
-
242
- if constexpr (!Ktraits::kIsEvenLen) { // So that the last state is correct
243
- if (threadIdx.x * kNItems + i >= params.seqlen - chunk * kChunkSize) {
235
+ if (seqlen % (kNItems * kNThreads) != 0) { // So that the last state is correct
236
+ if (threadIdx.x * kNItems + i >= seqlen - chunk * kChunkSize) {
237
thread_data[i] = make_float2(1.f, 0.f);
238
}
239
}
240
}
241
// Initialize running total
249
- scan_t running_prefix;
250
- // If we use WARP_SCAN then all lane 0 of all warps (not just thread 0) needs to read
251
- running_prefix = chunk == 0 ? x[(r * params.n_chunks) * params.dstate + state_idx] : ( threadIdx.x % 32 == 0 ? smem_running_prefix[state_idx + r * MAX_DSTATE] : make_float2(1.f, 0.f));
252
- // running_prefix = chunk > 0 && threadIdx.x == 0 ? smem_running_prefix[state_idx] : make_float2(1.f, 0.f);
242
+
243
+ scan_t running_prefix = chunk > 0 ? smem_running_prefix[state_idx + r * MAX_DSTATE] : make_float2(1.0, has_initial_state ? float(ssm_states[state_idx]): 0.0);
244
+
245
SSMScanPrefixCallbackOp<weight_t> prefix_op(running_prefix);
246
typename Ktraits::BlockScanT(smem_scan).InclusiveScan(
247
thread_data, thread_data, SSMScanOp<weight_t>(), prefix_op
@@ -258,7 +250,9 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
250
// Unless there's only 1 warp, but then it's the same thread (0) reading and writing.
251
if (threadIdx.x == 0) {
252
smem_running_prefix[state_idx] = prefix_op.running_prefix;
261
- x[(r * params.n_chunks + chunk) * params.dstate + state_idx] = prefix_op.running_prefix;
253
+ if (chunk == n_chunks - 1) {
254
+ ssm_states[state_idx] = input_t(prefix_op.running_prefix.y);
255
+ }
256
}
257
#pragma unroll
258
for (int i = 0; i < kNItems; ++i) {
@@ -270,7 +264,7 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
264
}
265
}
266
273
- input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
267
+ input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + sequence_start_index * params.out_batch_stride
268
+ dim_id * kNRows * params.out_d_stride + chunk * kChunkSize;
269
__syncthreads();
270
#pragma unroll
@@ -278,26 +272,26 @@ void selective_scan_fwd_kernel(SSMParamsBase params) {
272
if constexpr (!kDirectIO) {
273
if (r > 0) { __syncthreads(); }
274
}
281
- store_output<Ktraits>(out + r * params.out_d_stride, out_vals[r], smem_store, params.seqlen - chunk * kChunkSize);
275
+ store_output<Ktraits>(out + r * params.out_d_stride, out_vals[r], smem_store, seqlen - chunk * kChunkSize);
276
}
277
278
if constexpr (kHasZ) {
285
- input_t *z = reinterpret_cast<input_t *>(params.z_ptr) + batch_id * params.z_batch_stride
279
+ input_t *z = reinterpret_cast<input_t *>(params.z_ptr) + sequence_start_index * params.z_batch_stride
280
+ dim_id * kNRows * params.z_d_stride + chunk * kChunkSize;
287
- input_t *out_z = reinterpret_cast<input_t *>(params.out_z_ptr) + batch_id * params.out_z_batch_stride
281
+ input_t *out_z = reinterpret_cast<input_t *>(params.out_z_ptr) + sequence_start_index * params.out_z_batch_stride
282
+ dim_id * kNRows * params.out_z_d_stride + chunk * kChunkSize;
283
#pragma unroll
284
for (int r = 0; r < kNRows; ++r) {
285
input_t z_vals[kNItems];
286
__syncthreads();
293
- load_input<Ktraits>(z + r * params.z_d_stride, z_vals, smem_load, params.seqlen - chunk * kChunkSize);
287
+ load_input<Ktraits>(z + r * params.z_d_stride, z_vals, smem_load, seqlen - chunk * kChunkSize);
288
#pragma unroll
289
for (int i = 0; i < kNItems; ++i) {
290
float z_val = z_vals[i];
291
out_vals[r][i] *= z_val / (1 + expf(-z_val));
292
}
293
__syncthreads();
300
- store_output<Ktraits>(out_z + r * params.out_z_d_stride, out_vals[r], smem_store, params.seqlen - chunk * kChunkSize);
294
+ store_output<Ktraits>(out_z + r * params.out_z_d_stride, out_vals[r], smem_store, seqlen - chunk * kChunkSize);
295
}
296
}
297
@@ -316,8 +310,8 @@ void selective_scan_fwd_launch(SSMParamsBase ¶ms, cudaStream_t stream) {
310
constexpr bool kIsVariableC = true;
311
constexpr bool kHasZ = true;
312
BOOL_SWITCH(params.seqlen % (kNThreads * kNItems) == 0, kIsEvenLen, [&] {
319
- BOOL_SWITCH(params.index_ptr != nullptr , kUseIndex, [&] {
320
- using Ktraits = Selective_Scan_fwd_kernel_traits<kNThreads, kNItems, kNRows, kIsEvenLen, kIsVariableB, kIsVariableC, kHasZ, kUseIndex, input_t, weight_t>;
313
+ BOOL_SWITCH(params.query_start_loc_ptr != nullptr , kVarlen, [&] {
314
+ using Ktraits = Selective_Scan_fwd_kernel_traits<kNThreads, kNItems, kNRows, kIsEvenLen, kIsVariableB, kIsVariableC, kHasZ, kVarlen, input_t, weight_t>;
315
constexpr int kSmemSize = Ktraits::kSmemSize + kNRows * MAX_DSTATE * sizeof(typename Ktraits::scan_t);
316
dim3 grid(params.batch, params.dim / kNRows);
317
auto kernel = &selective_scan_fwd_kernel<Ktraits>;
@@ -405,12 +399,15 @@ void set_ssm_params_fwd(SSMParamsBase ¶ms,
399
const torch::Tensor out,
400
const torch::Tensor z,
401
const torch::Tensor out_z,
408
- void* D_ptr,
409
- void* delta_bias_ptr,
410
- void* x_ptr,
402
+ const c10::optional<at::Tensor>& D,
403
+ const c10::optional<at::Tensor>& delta_bias,
404
+ const torch::Tensor ssm_states,
405
bool has_z,
406
bool delta_softplus,
413
- void* index_ptr) {
407
+ const c10::optional<at::Tensor>& query_start_loc,
408
+ const c10::optional<at::Tensor>& cache_indices,
409
+ const c10::optional<at::Tensor>& has_initial_state,
410
+ bool varlen) {
411
412
// Reset the parameters
413
memset(¶ms, 0, sizeof(params));
@@ -434,55 +431,83 @@ void set_ssm_params_fwd(SSMParamsBase ¶ms,
431
params.A_ptr = A.data_ptr();
432
params.B_ptr = B.data_ptr();
433
params.C_ptr = C.data_ptr();
437
- params.D_ptr = D_ptr;
438
- params.delta_bias_ptr = delta_bias_ptr;
434
+ params.D_ptr = D.has_value() ? D.value().data_ptr() : nullptr;
435
+ params.delta_bias_ptr = delta_bias.has_value() ? delta_bias.value().data_ptr() : nullptr;
436
params.out_ptr = out.data_ptr();
440
- params.x_ptr = x_ptr;
437
+ params.ssm_states_ptr = ssm_states.data_ptr();
438
params.z_ptr = has_z ? z.data_ptr() : nullptr;
439
params.out_z_ptr = has_z ? out_z.data_ptr() : nullptr;
440
+ params.query_start_loc_ptr = query_start_loc.has_value() ? query_start_loc.value().data_ptr() : nullptr;
441
+ params.cache_indices_ptr = cache_indices.has_value() ? cache_indices.value().data_ptr() : nullptr;
442
+ params.has_initial_state_ptr = has_initial_state.has_value() ? has_initial_state.value().data_ptr() : nullptr;
443
444
- params.index_ptr = index_ptr;
444
445
// All stride are in elements, not bytes.
446
params.A_d_stride = A.stride(0);
447
params.A_dstate_stride = A.stride(1);
449
- if (!is_variable_B) {
450
- params.B_d_stride = B.stride(0);
451
- } else {
452
- params.B_batch_stride = B.stride(0);
453
- params.B_group_stride = B.stride(1);
454
- }
455
- params.B_dstate_stride = !is_variable_B ? B.stride(1) : B.stride(2);
456
- if (!is_variable_C) {
457
- params.C_d_stride = C.stride(0);
458
- } else {
459
- params.C_batch_stride = C.stride(0);
460
- params.C_group_stride = C.stride(1);
448
+
449
+ if (varlen){
450
+ params.B_batch_stride = B.stride(2);
451
+ params.B_group_stride = B.stride(0);
452
+ params.B_dstate_stride = B.stride(1);
453
+ params.C_batch_stride = C.stride(2);
454
+ params.C_group_stride = C.stride(0);
455
+ params.C_dstate_stride = C.stride(1);
456
+
457
+ params.u_batch_stride = u.stride(1);
458
+ params.u_d_stride = u.stride(0);
459
+ params.delta_batch_stride = delta.stride(1);
460
+ params.delta_d_stride = delta.stride(0);
461
+ if (has_z) {
462
+ params.z_batch_stride = z.stride(1);
463
+ params.z_d_stride = z.stride(0);
464
+ params.out_z_batch_stride = out_z.stride(1);
465
+ params.out_z_d_stride = out_z.stride(0);
466
+ }
467
+ params.out_batch_stride = out.stride(1);
468
+ params.out_d_stride = out.stride(0);
469
+
470
}
462
- params.C_dstate_stride = !is_variable_C ? C.stride(1) : C.stride(2);
463
- params.u_batch_stride = u.stride(0);
464
- params.u_d_stride = u.stride(1);
465
- params.delta_batch_stride = delta.stride(0);
466
- params.delta_d_stride = delta.stride(1);
467
- if (has_z) {
468
- params.z_batch_stride = z.stride(0);
469
- params.z_d_stride = z.stride(1);
470
- params.out_z_batch_stride = out_z.stride(0);
471
- params.out_z_d_stride = out_z.stride(1);
471
+ else{
472
+ if (!is_variable_B) {
473
+ params.B_d_stride = B.stride(0);
474
+ } else {
475
+ params.B_batch_stride = B.stride(0);
476
+ params.B_group_stride = B.stride(1);
477
+ }
478
+ params.B_dstate_stride = !is_variable_B ? B.stride(1) : B.stride(2);
479
+ if (!is_variable_C) {
480
+ params.C_d_stride = C.stride(0);
481
+ } else {
482
+ params.C_batch_stride = C.stride(0);
483
+ params.C_group_stride = C.stride(1);
484
+ }
485
+ params.C_dstate_stride = !is_variable_C ? C.stride(1) : C.stride(2);
486
+ params.u_batch_stride = u.stride(0);
487
+ params.u_d_stride = u.stride(1);
488
+ params.delta_batch_stride = delta.stride(0);
489
+ params.delta_d_stride = delta.stride(1);
490
+ if (has_z) {
491
+ params.z_batch_stride = z.stride(0);
492
+ params.z_d_stride = z.stride(1);
493
+ params.out_z_batch_stride = out_z.stride(0);
494
+ params.out_z_d_stride = out_z.stride(1);
495
+ }
496
+ params.out_batch_stride = out.stride(0);
497
+ params.out_d_stride = out.stride(1);
498
}
473
- params.out_batch_stride = out.stride(0);
474
- params.out_d_stride = out.stride(1);
499
}
500
477
-std::vector<torch::Tensor>
478
-selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
501
+void selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
502
const torch::Tensor &A, const torch::Tensor &B, const torch::Tensor &C,
503
const c10::optional<torch::Tensor> &D_,
504
const c10::optional<torch::Tensor> &z_,
505
const c10::optional<torch::Tensor> &delta_bias_,
506
bool delta_softplus,
484
- const c10::optional<torch::Tensor> &index_,
485
- const c10::optional<torch::Tensor> &x) {
507
+ const c10::optional<torch::Tensor> &query_start_loc,
508
+ const c10::optional<torch::Tensor> &cache_indices,
509
+ const c10::optional<torch::Tensor> &has_initial_state,
510
+ const torch::Tensor &ssm_states) {
511
auto input_type = u.scalar_type();
512
auto weight_type = A.scalar_type();
513
TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
@@ -505,23 +530,37 @@ selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
530
TORCH_CHECK(delta.stride(-1) == 1 || delta.size(-1) == 1);
531
532
const auto sizes = u.sizes();
508
- const int batch_size = sizes[0];
509
- const int dim = sizes[1];
510
- const int seqlen = sizes[2];
533
+ const bool varlen = query_start_loc.has_value();
534
+ const int batch_size = varlen ? query_start_loc.value().sizes()[0] - 1 : sizes[0];
535
+ const int dim = varlen ? sizes[0] : sizes[1];
536
+ const int seqlen = varlen ? sizes[1] : sizes[2];
537
const int dstate = A.size(1);
512
- const int n_groups = is_variable_B ? B.size(1) : 1;
538
+ const int n_groups = varlen ? B.size(0) : B.size(1);
539
540
TORCH_CHECK(dstate <= 256, "selective_scan only supports state dimension <= 256");
541
516
- CHECK_SHAPE(u, batch_size, dim, seqlen);
517
- CHECK_SHAPE(delta, batch_size, dim, seqlen);
542
+ if (varlen) {
543
+ CHECK_SHAPE(u, dim, seqlen);
544
+ CHECK_SHAPE(delta, dim, seqlen);
545
+ } else {
546
+ CHECK_SHAPE(u, batch_size, dim, seqlen);
547
+ CHECK_SHAPE(delta, batch_size, dim, seqlen);
548
+ }
549
CHECK_SHAPE(A, dim, dstate);
550
TORCH_CHECK(is_variable_B, "is_variable_B = False is disabled in favor of reduced binary size")
520
- CHECK_SHAPE(B, batch_size, n_groups, dstate, seqlen );
551
+ if (varlen) {
552
+ CHECK_SHAPE(B, n_groups, dstate, seqlen);
553
+ } else {
554
+ CHECK_SHAPE(B, batch_size, n_groups, dstate, seqlen);
555
+ }
556
TORCH_CHECK(B.stride(-1) == 1 || B.size(-1) == 1);
557
558
TORCH_CHECK(is_variable_C, "is_variable_C = False is disabled in favor of reduced binary size")
524
- CHECK_SHAPE(C, batch_size, n_groups, dstate, seqlen);
559
+ if (varlen) {
560
+ CHECK_SHAPE(C, n_groups, dstate, seqlen);
561
+ } else {
562
+ CHECK_SHAPE(C, batch_size, n_groups, dstate, seqlen);
563
+ }
564
TORCH_CHECK(C.stride(-1) == 1 || C.size(-1) == 1);
565
566
if (D_.has_value()) {
@@ -539,12 +578,30 @@ selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
578
TORCH_CHECK(delta_bias.stride(-1) == 1 || delta_bias.size(-1) == 1);
579
CHECK_SHAPE(delta_bias, dim);
580
}
542
- if (index_.has_value()) {
543
- auto index = index_.value();
544
- TORCH_CHECK(index.scalar_type() == at::ScalarType::Int);
545
- TORCH_CHECK(index.is_cuda());
546
- CHECK_SHAPE(index, batch_size, seqlen);
581
+
582
+
583
+ if (has_initial_state.has_value()) {
584
+ auto has_initial_state_ = has_initial_state.value();
585
+ TORCH_CHECK(has_initial_state_.scalar_type() == at::ScalarType::Bool);
586
+ TORCH_CHECK(has_initial_state_.is_cuda());
587
+ CHECK_SHAPE(has_initial_state_, batch_size);
588
+ }
589
+
590
+
591
+ if (query_start_loc.has_value()) {
592
+ auto query_start_loc_ = query_start_loc.value();
593
+ TORCH_CHECK(query_start_loc_.scalar_type() == at::ScalarType::Int);
594
+ TORCH_CHECK(query_start_loc_.is_cuda());
595
+ }
596
+
597
+
598
+ if (cache_indices.has_value()) {
599
+ auto cache_indices_ = cache_indices.value();
600
+ TORCH_CHECK(cache_indices_.scalar_type() == at::ScalarType::Int);
601
+ TORCH_CHECK(cache_indices_.is_cuda());
602
+ CHECK_SHAPE(cache_indices_, batch_size);
603
}
604
+
605
606
at::Tensor z, out_z;
607
const bool has_z = z_.has_value();
@@ -553,32 +610,39 @@ selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
610
TORCH_CHECK(z.scalar_type() == input_type);
611
TORCH_CHECK(z.is_cuda());
612
TORCH_CHECK(z.stride(-1) == 1 || z.size(-1) == 1);
556
- CHECK_SHAPE(z, batch_size, dim, seqlen);
557
- out_z = torch::empty_like(z);
613
+ if (varlen){
614
+ CHECK_SHAPE(z, dim, seqlen);
615
+ } else {
616
+ CHECK_SHAPE(z, batch_size, dim, seqlen);
617
+ }
618
+
619
+ out_z = z;
620
621
const int n_chunks = (seqlen + 2048 - 1) / 2048;
622
// const int n_chunks = (seqlen + 1024 - 1) / 1024;
623
// at::Tensor out = torch::empty_like(u);
624
// Right now u has BHL layout and delta has HBL layout, and we want out to have HBL layout
563
- at::Tensor out = torch::empty_like(delta);
564
- if (x.has_value()){
565
- auto _x = x.value();
566
- TORCH_CHECK(_x.scalar_type() == weight_type);
567
- TORCH_CHECK(_x.is_cuda());
568
- TORCH_CHECK(_x.stride(-1) == 1);
569
- CHECK_SHAPE(_x, batch_size, dim, n_chunks, dstate * 2);
570
- }
625
+ at::Tensor out = delta;
626
+ TORCH_CHECK(ssm_states.scalar_type() == input_type);
627
+ TORCH_CHECK(ssm_states.is_cuda());
628
+ TORCH_CHECK(ssm_states.stride(-1) == 1);
629
+ CHECK_SHAPE(ssm_states, batch_size, dim, dstate);
630
631
SSMParamsBase params;
632
set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, n_chunks, is_variable_B, is_variable_C,
633
u, delta, A, B, C, out, z, out_z,
575
- D_.has_value() ? D_.value().data_ptr() : nullptr,
576
- delta_bias_.has_value() ? delta_bias_.value().data_ptr() : nullptr,
577
- x.value().data_ptr(),
634
+ D_,
635
+ delta_bias_,
636
+ ssm_states,
637
has_z,
638
delta_softplus,
580
- index_.has_value() ? index_.value().data_ptr() : nullptr);
639
+ query_start_loc,
640
+ cache_indices,
641
+ has_initial_state,
642
+ varlen
643
+ );
644
645
+
646
// Otherwise the kernel will be launched from cuda:0 device
647
// Cast to char to avoid compiler warning about narrowing
648
at::cuda::CUDAGuard device_guard{(char)u.get_device()};
@@ -586,8 +650,5 @@ selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
650
DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(u.scalar_type(), "selective_scan_fwd", [&] {
651
selective_scan_fwd_cuda<input_t, weight_t>(params, stream);
652
});
589
- std::vector<at::Tensor> result = {out};
590
- if (has_z) { result.push_back(out_z); }
591
- return result;
653
}
654
csrc/ops.h
+18
-13
@@ -215,25 +215,30 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
215
torch::Tensor experts_ids,
216
torch::Tensor num_tokens_post_pad);
217
218
-std::vector<torch::Tensor> selective_scan_fwd(
219
- const torch::Tensor& u, const torch::Tensor& delta, const torch::Tensor& A,
220
- const torch::Tensor& B, const torch::Tensor& C,
221
- const c10::optional<torch::Tensor>& D_,
222
- const c10::optional<torch::Tensor>& z_,
223
- const c10::optional<torch::Tensor>& delta_bias_, bool delta_softplus,
224
- const c10::optional<torch::Tensor>& index_,
225
- const c10::optional<torch::Tensor>& x);
218
+void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
219
+ const torch::Tensor& A, const torch::Tensor& B,
220
+ const torch::Tensor& C,
221
+ const c10::optional<torch::Tensor>& D_,
222
+ const c10::optional<torch::Tensor>& z_,
223
+ const c10::optional<torch::Tensor>& delta_bias_,
224
+ bool delta_softplus,
225
+ const c10::optional<torch::Tensor>& query_start_loc,
226
+ const c10::optional<torch::Tensor>& cache_indices,
227
+ const c10::optional<torch::Tensor>& has_initial_state,
228
+ const torch::Tensor& ssm_states);
229
230
at::Tensor causal_conv1d_update(
231
const at::Tensor& x, const at::Tensor& conv_state, const at::Tensor& weight,
229
- const c10::optional<at::Tensor>& bias, bool silu_activation,
230
- const c10::optional<at::Tensor>& conv_state_indices);
232
+ const c10::optional<at::Tensor>& bias_, bool silu_activation,
233
+ const c10::optional<at::Tensor>& cache_seqlens_,
234
+ const c10::optional<at::Tensor>& conv_state_indices_);
235
236
at::Tensor causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
237
const c10::optional<at::Tensor>& bias_,
234
- const c10::optional<at::Tensor>& seq_idx_,
235
- const c10::optional<at::Tensor>& initial_states_,
236
- const c10::optional<at::Tensor>& final_states_out_,
238
+ const c10::optional<at::Tensor>& conv_states,
239
+ const c10::optional<at::Tensor>& query_start_loc,
240
+ const c10::optional<at::Tensor>& cache_indices,
241
+ const c10::optional<at::Tensor>& has_initial_state,
242
bool silu_activation);
243
244
#ifndef USE_ROCM
csrc/torch_bindings.cpp
+11
-6
@@ -273,26 +273,31 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
273
ops.def(
274
"selective_scan_fwd(Tensor! u, Tensor! delta,"
275
"Tensor! A, Tensor! B, Tensor! C,"
276
- "Tensor? D_, Tensor? z_, Tensor? delta_bias_,"
276
+ "Tensor? D_, Tensor!? z_, Tensor? delta_bias_,"
277
"bool delta_softplus,"
278
- "Tensor? index_, Tensor!? x) -> Tensor[]");
278
+ "Tensor? query_start_loc,"
279
+ "Tensor? cache_indices,"
280
+ "Tensor? has_initial_state,"
281
+ "Tensor! ssm_states) -> ()");
282
ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
283
284
ops.def(
285
"causal_conv1d_update(Tensor! x,"
286
"Tensor! conv_state,"
287
"Tensor! weight,"
285
- "Tensor? bias,"
288
+ "Tensor? bias_,"
289
"bool silu_activation,"
290
+ "Tensor? cache_seqlens_,"
291
"Tensor? conv_state_indices) -> Tensor");
292
ops.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update);
293
294
ops.def(
295
"causal_conv1d_fwd(Tensor! x, Tensor! weight,"
296
"Tensor? bias_,"
293
- "Tensor? seq_idx_,"
294
- "Tensor? initial_states_,"
295
- "Tensor!? final_states_out_,"
297
+ "Tensor!? conv_states,"
298
+ "Tensor? query_start_loc,"
299
+ "Tensor? cache_indices,"
300
+ "Tensor? has_initial_state,"
301
"bool silu_activation) -> Tensor");
302
ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
303
#endif
tests/kernels/test_causal_conv1d.py
+218
-128
@@ -3,7 +3,6 @@ from typing import Optional
3
import pytest
4
import torch
5
import torch.nn.functional as F
6
-from einops import rearrange
6
7
from tests.kernels.utils import opcheck
8
from vllm import _custom_ops as ops # noqa: F401
@@ -57,43 +56,72 @@ def causal_conv1d_ref(
56
return (out, None) if not return_final_states else (out, final_states_out)
57
58
60
-def causal_conv1d_update_ref(x: torch.Tensor,
61
- conv_state: torch.Tensor,
62
- weight: torch.Tensor,
63
- bias: Optional[torch.Tensor] = None,
64
- activation: Optional[str] = None):
59
+def causal_conv1d_update_ref(x,
60
+ conv_state,
61
+ weight,
62
+ bias=None,
63
+ activation=None,
64
+ cache_seqlens=None):
65
"""
66
- x: (batch, dim)
67
- conv_state: (batch, dim, width)
66
+ x: (batch, dim) or (batch, dim, seqlen)
67
+ conv_state: (batch, dim, state_len), where state_len >= width - 1
68
weight: (dim, width)
69
bias: (dim,)
70
+ cache_seqlens: (batch,), dtype int32.
71
+ If not None, the conv_state is treated as a circular buffer.
72
+ The conv_state will be updated by copying x to the
73
+ conv_state starting at the index
74
+ @cache_seqlens % state_len before performing the convolution.
75
71
- out: (batch, dim)
76
+ out: (batch, dim) or (batch, dim, seqlen)
77
"""
78
if activation not in [None, "silu", "swish"]:
79
raise NotImplementedError("activation must be None, silu, or swish")
80
dtype_in = x.dtype
76
- batch, dim = x.shape
81
+ unsqueeze = x.dim() == 2
82
+ if unsqueeze:
83
+ x = x.unsqueeze(-1)
84
+ batch, dim, seqlen = x.shape
85
width = weight.shape[1]
78
- assert conv_state.shape == (batch, dim, width)
86
+ state_len = conv_state.shape[-1]
87
+ assert conv_state.shape == (batch, dim, state_len)
88
assert weight.shape == (dim, width)
80
- conv_state.copy_(torch.roll(conv_state, shifts=-1,
81
- dims=-1)) # Update state (B D W)
82
- conv_state[:, :, -1] = x
83
- out = torch.sum(conv_state * weight, dim=-1) # (B D)
84
- if bias is not None:
85
- out += bias
89
+ if cache_seqlens is None:
90
+ x_new = torch.cat([conv_state, x], dim=-1).to(
91
+ weight.dtype) # (batch, dim, state_len + seqlen)
92
+ conv_state.copy_(x_new[:, :, -state_len:])
93
+ else:
94
+ width_idx = torch.arange(
95
+ -(width - 1), 0, dtype=torch.long,
96
+ device=x.device).unsqueeze(0) + cache_seqlens.unsqueeze(1)
97
+ width_idx = torch.remainder(width_idx, state_len).unsqueeze(1).expand(
98
+ -1, dim, -1)
99
+ x_new = torch.cat([conv_state.gather(2, width_idx), x],
100
+ dim=-1).to(weight.dtype)
101
+ copy_idx = torch.arange(
102
+ seqlen, dtype=torch.long,
103
+ device=x.device).unsqueeze(0) + cache_seqlens.unsqueeze(1)
104
+ copy_idx = torch.remainder(copy_idx,
105
+ state_len).unsqueeze(1).expand(-1, dim, -1)
106
+ conv_state.scatter_(2, copy_idx, x)
107
+ out = F.conv1d(x_new, weight.unsqueeze(1), bias, padding=0,
108
+ groups=dim)[:, :, -seqlen:]
109
+ if unsqueeze:
110
+ out = out.squeeze(-1)
111
return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
112
113
114
+@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
115
+@pytest.mark.parametrize("silu_activation", [True])
116
+@pytest.mark.parametrize("has_bias", [True])
117
def causal_conv1d_opcheck_fn(
118
x: torch.Tensor,
119
weight: torch.Tensor,
120
bias: Optional[torch.Tensor] = None,
93
- seq_idx: Optional[torch.Tensor] = None,
94
- initial_states: Optional[torch.Tensor] = None,
95
- return_final_states: bool = False,
96
- final_states_out=None,
121
+ cu_seq_len: Optional[torch.Tensor] = None,
122
+ cache_indices: Optional[torch.Tensor] = None,
123
+ has_initial_state: Optional[torch.Tensor] = None,
124
+ conv_states: Optional[torch.Tensor] = None,
125
activation: Optional[str] = "silu",
126
):
127
"""
@@ -109,135 +137,93 @@ def causal_conv1d_opcheck_fn(
137
"""
138
if activation not in [None, "silu", "swish"]:
139
raise NotImplementedError("activation must be None, silu, or swish")
112
- if x.stride(2) != 1 and x.stride(1) != 1:
140
+ if x.stride(-1) != 1:
141
x = x.contiguous()
142
bias = bias.contiguous() if bias is not None else None
115
- if seq_idx is not None:
116
- assert (initial_states is
117
- None), "initial_states must be None if seq_idx is not None"
118
- assert (not return_final_states
119
- ), "If seq_idx is not None, we don't return final_states_out"
120
- seq_idx = seq_idx.contiguous() if seq_idx is not None else None
121
- if initial_states is not None and (initial_states.stride(2) != 1
122
- and initial_states.stride(1) != 1):
123
- initial_states = initial_states.contiguous()
124
- if return_final_states:
125
- assert (
126
- x.stride(1) == 1
127
- ), "Only channel-last layout support returning final_states_out"
128
- if final_states_out is not None:
129
- assert (final_states_out.stride(2) == 1
130
- or final_states_out.stride(1) == 1)
131
- else:
132
- batch, dim, seqlen = x.shape
133
- width = weight.shape[1]
134
- final_states_out = torch.empty(batch,
135
- width - 1,
136
- dim,
137
- device=x.device,
138
- dtype=x.dtype).transpose(1, 2)
139
- else:
140
- final_states_out = None
143
142
- opcheck(torch.ops._C.causal_conv1d_fwd,
143
- (x, weight, bias, seq_idx, initial_states, final_states_out,
144
- activation in ["silu", "swish"]))
144
+ opcheck(torch.ops._C.causal_conv1d_fwd, (
145
+ x,
146
+ weight,
147
+ bias,
148
+ conv_states,
149
+ cu_seq_len,
150
+ cache_indices,
151
+ has_initial_state,
152
+ activation in ["silu", "swish"],
153
+ ))
154
155
147
-@pytest.mark.parametrize("return_final_states", [False, True])
148
-@pytest.mark.parametrize("has_initial_states", [False, True])
149
-@pytest.mark.parametrize("channel_last", [False, True])
150
-@pytest.mark.parametrize("itype", [torch.bfloat16])
151
-@pytest.mark.parametrize("silu_activation", [False, True])
152
-@pytest.mark.parametrize("has_bias", [False, True])
156
+@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float])
157
+@pytest.mark.parametrize("silu_activation", [True])
158
+@pytest.mark.parametrize("has_bias", [True])
159
@pytest.mark.parametrize("width", [4])
154
-@pytest.mark.parametrize("seqlen", [128, 512, 4096])
155
-@pytest.mark.parametrize('dim', [64, 4096 + 32])
156
-@pytest.mark.parametrize('batch', [1, 2])
160
+@pytest.mark.parametrize(
161
+ 'seqlen', [1, 8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 4096])
162
+@pytest.mark.parametrize('dim', [64])
163
+@pytest.mark.parametrize('batch', [1])
164
def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
158
- itype, channel_last, has_initial_states,
159
- return_final_states):
160
- if not channel_last and (has_initial_states or return_final_states):
161
- pytest.skip(
162
- "Only channel_last support initial_states or return_final_states")
165
+ itype):
166
device = "cuda"
167
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
168
if itype == torch.bfloat16:
169
rtol, atol = 1e-2, 5e-2
170
# set seed
171
seed_everything(0)
169
- if not channel_last:
170
- x = torch.randn(batch,
171
- 4096 + dim + 64,
172
- seqlen,
173
- device=device,
174
- dtype=itype)[:, 4096:4096 + dim, :]
175
- else:
176
- x = rearrange(
177
- torch.randn(batch,
178
- seqlen,
179
- 4096 + dim + 64,
180
- device=device,
181
- dtype=itype)[:, :, 4096:4096 + dim], "b s d -> b d s")
172
+ x = torch.randn(batch, dim, seqlen, device=device,
173
+ dtype=itype).contiguous()
174
+
175
weight = torch.randn(dim, width, device=device, dtype=itype)
176
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
184
- if has_initial_states:
185
- initial_states = torch.randn(batch,
186
- width - 1,
187
- dim,
188
- device=device,
189
- dtype=itype).transpose(1, 2)
190
- else:
191
- initial_states = None
192
- x_ref = x.detach().clone()
193
- weight_ref = weight.detach().clone()
194
- bias_ref = bias.detach().clone() if bias is not None else None
195
- initial_states_ref = initial_states.detach().clone(
177
+ initial_states = torch.randn(batch,
178
+ dim,
179
+ width - 1,
180
+ device=device,
181
+ dtype=itype)
182
+ x_ref = x.clone()
183
+ weight_ref = weight.clone()
184
+ bias_ref = bias.clone() if bias is not None else None
185
+ initial_states_ref = initial_states.clone(
186
) if initial_states is not None else None
187
activation = None if not silu_activation else "silu"
198
- out, final_states = causal_conv1d_fn(
199
- x,
200
- weight,
201
- bias,
202
- initial_states=initial_states,
203
- return_final_states=return_final_states,
204
- activation=activation)
188
+ out = causal_conv1d_fn(x,
189
+ weight,
190
+ bias,
191
+ activation=activation,
192
+ conv_states=initial_states,
193
+ has_initial_state=torch.ones(batch,
194
+ dtype=torch.bool,
195
+ device=x.device))
196
out_ref, final_states_ref = causal_conv1d_ref(
197
x_ref,
198
weight_ref,
199
bias_ref,
200
initial_states=initial_states_ref,
210
- return_final_states=return_final_states,
201
+ return_final_states=True,
202
activation=activation)
212
-
213
- causal_conv1d_opcheck_fn(x_ref,
214
- weight_ref,
215
- bias_ref,
216
- initial_states=initial_states_ref,
217
- return_final_states=return_final_states,
218
- activation=activation)
219
-
220
- if return_final_states:
221
- assert final_states is not None and final_states_ref is not None
222
- assert torch.allclose(final_states,
223
- final_states_ref,
224
- rtol=rtol,
225
- atol=atol)
226
-
203
+ assert initial_states is not None and final_states_ref is not None
204
+ assert torch.allclose(initial_states,
205
+ final_states_ref,
206
+ rtol=rtol,
207
+ atol=atol)
208
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
209
229
- if return_final_states:
230
- out += F.sigmoid(final_states).sum(dim=-1, keepdim=True)
231
- out_ref += F.sigmoid(final_states_ref).sum(dim=-1, keepdim=True)
210
+ causal_conv1d_opcheck_fn(x,
211
+ weight,
212
+ bias,
213
+ activation=activation,
214
+ conv_states=initial_states,
215
+ has_initial_state=torch.ones(batch,
216
+ dtype=torch.bool,
217
+ device=x.device))
218
219
220
@pytest.mark.parametrize("itype", [torch.bfloat16])
221
@pytest.mark.parametrize("silu_activation", [False, True])
222
@pytest.mark.parametrize("has_bias", [False, True])
237
-@pytest.mark.parametrize("width", [2, 3, 4])
223
+@pytest.mark.parametrize("seqlen", [1])
224
+@pytest.mark.parametrize("width", [4])
225
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
239
-@pytest.mark.parametrize("batch", [1, 2])
240
-def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
226
+def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation,
227
itype):
228
device = "cuda"
229
rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
@@ -246,8 +232,9 @@ def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
232
# set seed
233
seed_everything(0)
234
batch = 2
249
- x = torch.randn(batch, dim, device=device, dtype=itype)
250
- conv_state = torch.randn(batch, dim, width, device=device, dtype=itype)
235
+ x = torch.randn(batch, dim, seqlen, device=device, dtype=itype)
236
+ conv_state = torch.randn(batch, dim, width - 1, device=device, dtype=itype)
237
+
238
weight = torch.randn(dim,
239
width,
240
device=device,
@@ -273,9 +260,15 @@ def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
260
assert torch.equal(conv_state, conv_state_ref)
261
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
262
276
- opcheck(
277
- torch.ops._C.causal_conv1d_update,
278
- (x, conv_state, weight, bias, activation in ["silu", "swish"], None))
263
+ opcheck(torch.ops._C.causal_conv1d_update, (
264
+ x,
265
+ conv_state,
266
+ weight,
267
+ bias,
268
+ activation in ["silu", "swish"],
269
+ None,
270
+ None,
271
+ ))
272
273
274
@pytest.mark.parametrize("itype",
@@ -292,16 +285,16 @@ def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
285
if itype == torch.bfloat16:
286
rtol, atol = 1e-2, 5e-2
287
295
- # set seed
296
- torch.random.manual_seed(0)
288
+ # set )seed
289
+ seed_everything(0)
290
batch = 64
291
299
- x = torch.randn(batch, dim, device=device, dtype=itype)
292
+ x = torch.randn(batch, dim, 1, device=device, dtype=itype)
293
294
total_entries = 10 * batch
295
conv_state = torch.randn(total_entries,
296
dim,
304
- width,
297
+ width - 1,
298
device=device,
299
dtype=itype)
300
conv_state_indices = torch.randperm(total_entries)[:batch].to(
@@ -332,3 +325,100 @@ def test_causal_conv1d_update_with_batch_gather(dim, width, seqlen, has_bias,
325
326
assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref)
327
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
328
+
329
+ opcheck(torch.ops._C.causal_conv1d_update, (
330
+ x,
331
+ conv_state,
332
+ weight,
333
+ bias,
334
+ activation in ["silu", "swish"],
335
+ None,
336
+ conv_state_indices,
337
+ ))
338
+
339
+
340
+@pytest.mark.parametrize("itype", [torch.bfloat16])
341
+@pytest.mark.parametrize("silu_activation", [True])
342
+@pytest.mark.parametrize("has_bias", [True])
343
+@pytest.mark.parametrize("width", [4])
344
+@pytest.mark.parametrize('seqlen',
345
+ [8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 4096])
346
+@pytest.mark.parametrize('dim', [64, 4096])
347
+def test_causal_conv1d_varlen(dim, seqlen, width, has_bias, silu_activation,
348
+ itype):
349
+ device = "cuda"
350
+ rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
351
+ if itype == torch.bfloat16:
352
+ rtol, atol = 1e-2, 5e-2
353
+ # set seed
354
+ seed_everything(0)
355
+ batch = 1
356
+ seqlens = []
357
+ nsplits = 3
358
+ eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
359
+ seqlens.append(
360
+ torch.diff(
361
+ torch.cat(
362
+ [torch.tensor([-1]), eos_pos,
363
+ torch.tensor([seqlen - 1])])).tolist())
364
+ assert sum(seqlens[-1]) == seqlen
365
+ assert all(s > 0 for s in seqlens[-1])
366
+
367
+ cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
368
+ cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum],
369
+ dim=0)
370
+ x = torch.randn(batch, 4096 + dim + 64, seqlen, device=device,
371
+ dtype=itype)[:, 4096:4096 + dim, :]
372
+ weight = torch.randn(dim, width, device=device, dtype=itype)
373
+ bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
374
+ x_ref = x.clone()
375
+ weight_ref = weight.clone()
376
+ bias_ref = bias.clone() if bias is not None else None
377
+ activation = None if not silu_activation else "silu"
378
+ final_states = torch.randn(nsplits + 1,
379
+ dim,
380
+ width - 1,
381
+ device=x.device,
382
+ dtype=x.dtype)
383
+ final_states_ref = final_states.clone()
384
+ has_initial_states = torch.randint(0,
385
+ 2, (cumsum.shape[0] - 1, ),
386
+ dtype=torch.bool,
387
+ device=x.device)
388
+ cache_indices = torch.randperm(cumsum.shape[0] - 1,
389
+ dtype=torch.int32,
390
+ device=x.device)
391
+ out = causal_conv1d_fn(x.squeeze(0), weight, bias, cumsum.cuda(),
392
+ cache_indices, has_initial_states, final_states,
393
+ activation)
394
+ out_ref = []
395
+ out_ref_b = []
396
+
397
+ splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)]
398
+ for i in range(len(seqlens[0])):
399
+ x_s = [v[i].unsqueeze(0) for v in splits][0]
400
+ out_ref_b.append(
401
+ causal_conv1d_ref(
402
+ x_s,
403
+ weight_ref,
404
+ bias_ref,
405
+ activation=activation,
406
+ return_final_states=True,
407
+ final_states_out=final_states_ref[cache_indices[i]].unsqueeze(
408
+ 0),
409
+ initial_states=final_states_ref[cache_indices[i]].unsqueeze(0)
410
+ if has_initial_states[i] else None))
411
+ out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2))
412
+ out_ref = torch.cat(out_ref, dim=0)
413
+
414
+ print(f"Output max diff: {(out - out_ref).abs().max().item()}")
415
+ print(f"Output mean diff: {(out - out_ref).abs().mean().item()}")
416
+ print("Output state max diff"
417
+ f":{(final_states - final_states_ref).abs().max()}")
418
+ print("Output state mean diff"
419
+ f":{(final_states - final_states_ref).abs().mean()}")
420
+ assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
421
+ assert torch.allclose(final_states, final_states_ref, rtol=rtol, atol=atol)
422
+ causal_conv1d_opcheck_fn(x.squeeze(0), weight, bias, cumsum.cuda(),
423
+ cache_indices, has_initial_states, final_states,
424
+ activation)
tests/kernels/test_mamba_ssm.py
+198
-69
@@ -98,8 +98,8 @@ def selective_scan_ref(u,
98
delta_bias=None,
99
delta_softplus=False,
100
return_last_state=False,
101
- position_indices=None,
102
- prev_state=None):
101
+ prev_state=None,
102
+ final_state_out=None):
103
"""
104
u: r(B D L)
105
delta: r(B D L)
@@ -139,12 +139,8 @@ def selective_scan_ref(u,
139
deltaB_u = torch.einsum('bdl,bdnl,bdl->bdln', delta, B, u)
140
if is_variable_C and C.dim() == 4:
141
C = repeat(C, "B G N L -> B (G H) N L", H=dim // C.shape[1])
142
- last_state = None
142
for i in range(u.shape[2]):
144
- if position_indices is not None and position_indices[0, i] == 0:
145
- x = deltaB_u[:, :, i]
146
- else:
147
- x = deltaA[:, :, i] * x + deltaB_u[:, :, i]
143
+ x = deltaA[:, :, i] * x + deltaB_u[:, :, i]
144
if not is_variable_C:
145
y = torch.einsum('bdn,dn->bd', x, C)
146
else:
@@ -153,14 +149,17 @@ def selective_scan_ref(u,
149
else:
150
y = torch.einsum('bdn,bdn->bd', x, C[:, :, :, i])
151
if i == u.shape[2] - 1:
156
- last_state = x
152
+ if final_state_out is None:
153
+ final_state_out = x
154
+ else:
155
+ final_state_out.copy_(x)
156
ys.append(y)
157
y = torch.stack(ys, dim=2) # (batch dim L)
158
out = y if D is None else y + u * rearrange(D, "d -> d 1")
159
if z is not None:
160
out = out * F.silu(z)
161
out = out.to(dtype=dtype_in)
163
- return out if not return_last_state else (out, last_state)
162
+ return out if not return_last_state else (out, final_state_out)
163
164
165
def selective_scan_opcheck_fn(u,
@@ -172,9 +171,10 @@ def selective_scan_opcheck_fn(u,
171
z=None,
172
delta_bias=None,
173
delta_softplus=False,
175
- return_last_state=False,
176
- position_indices=None,
177
- prev_state=None):
174
+ cu_seq_len=None,
175
+ cache_indices=None,
176
+ has_initial_state=None,
177
+ ssm_states=None):
178
"""if return_last_state is True, returns (out, last_state)
179
last_state has shape (batch, dim, dstate).
180
"""
@@ -190,36 +190,27 @@ def selective_scan_opcheck_fn(u,
190
C = C.contiguous()
191
if z is not None and z.stride(-1) != 1:
192
z = z.contiguous()
193
- if B.dim() == 3:
193
+ if B.dim() == 3 and cu_seq_len is None:
194
B = B.unsqueeze(1)
195
- if C.dim() == 3:
195
+ if B.dim() == 2 and cu_seq_len is not None:
196
+ B = B.unsqueeze(0)
197
+ if C.dim() == 3 and cu_seq_len is None:
198
C = C.unsqueeze(1)
197
- n_chunks = int((u.shape[-1] + 2048 - 1) / 2048)
198
- x = torch.zeros((
199
- u.shape[0],
200
- u.shape[1],
201
- n_chunks,
202
- int(A.shape[1] * 2),
203
- ),
204
- device=u.device,
205
- dtype=torch.float32,
206
- requires_grad=False)
207
- x[:, :, 0, 0::2] = 1
208
- if prev_state is not None:
209
- x[:, :, 0, 1::2].copy_(prev_state)
199
+ if C.dim() == 2 and cu_seq_len is not None:
200
+ C = C.unsqueeze(0)
201
202
# Disable test_autograd_registration for now as it seems to trigger
203
# a bogus error.
204
opcheck(torch.ops._C.selective_scan_fwd,
214
- (u, delta, A, B, C, D, z, delta_bias, delta_softplus,
215
- position_indices, x),
205
+ (u, delta, A, B, C, D, z, delta_bias, delta_softplus, cu_seq_len,
206
+ cache_indices, has_initial_state, ssm_states),
207
test_utils=["test_schema", "test_faketensor"])
208
209
210
@pytest.mark.parametrize('wtype', [torch.float32])
220
-@pytest.mark.parametrize('itype', [torch.float32])
211
+@pytest.mark.parametrize('itype',
212
+ [torch.float32, torch.float16, torch.bfloat16])
213
@pytest.mark.parametrize('seqlen', [128, 256, 512, 1024, 2048, 4096])
222
-@pytest.mark.parametrize("return_last_state", [True])
214
@pytest.mark.parametrize('has_delta_bias', [True])
215
@pytest.mark.parametrize('delta_softplus', [True])
216
@pytest.mark.parametrize('has_z', [True])
@@ -229,8 +220,8 @@ def selective_scan_opcheck_fn(u,
220
@pytest.mark.parametrize("is_variable_B", [True])
221
@pytest.mark.parametrize("scan_chunks", [1, 2, 3])
222
def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
232
- has_z, has_delta_bias, delta_softplus,
233
- return_last_state, seqlen, itype, wtype, scan_chunks):
223
+ has_z, has_delta_bias, delta_softplus, seqlen, itype,
224
+ wtype, scan_chunks):
225
if varBC_groups > 1 and (not is_variable_B or not is_variable_C):
226
pytest.skip() # This config is not applicable
227
device = 'cuda'
@@ -243,10 +234,11 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
234
atolw = max(atolw, atol)
235
# set seed
236
seed_everything(0)
246
- batch_size = 2
237
+ batch_size = 1
238
dim = 4
239
dstate = 8
240
A = (-0.5 * torch.rand(dim, dstate, device=device, dtype=wtype))
241
+ A_ref = A.clone()
242
if not is_variable_B:
243
B_shape = [dim, dstate]
244
elif varBC_groups == 1:
@@ -256,6 +248,7 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
248
B = torch.randn(B_shape,
249
device=device,
250
dtype=wtype if not is_variable_B else itype)
251
+ B_ref = B.clone()
252
if not is_variable_C:
253
C_shape = [dim, dstate]
254
elif varBC_groups == 1:
@@ -265,16 +258,25 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
258
C = torch.randn(C_shape,
259
device=device,
260
dtype=wtype if not is_variable_C else itype)
261
+ C_ref = C.clone()
262
D = torch.randn(dim, device=device, dtype=torch.float32) if has_D else None
263
+ D_ref = D.clone()
264
z = torch.randn(batch_size, dim, seqlen, device=device,
265
dtype=itype) if has_z else None
266
+ z_ref = z.clone() if has_z else None
267
delta_bias = (0.5 * torch.rand(dim, device=device, dtype=torch.float32)
268
) if has_delta_bias else None
269
u = torch.randn(batch_size, dim, seqlen, device=device, dtype=itype)
270
+ u_ref = u.clone()
271
delta = (0.5 *
272
torch.rand(batch_size, dim, seqlen, device=device, dtype=itype))
276
- state = None
277
- state_ref = None
273
+ delta_ref = delta.clone()
274
+ state_shape = (batch_size, u.shape[1], int(A.shape[1]))
275
+ state = torch.randn(state_shape,
276
+ device=u.device,
277
+ dtype=itype,
278
+ requires_grad=False)
279
+ state_ref = state.clone()
280
out = None
281
out_ref = None
282
outs = []
@@ -294,40 +296,40 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
296
if has_z:
297
assert z is not None
298
_z = z[..., chunk_start:chunk_end]
297
- out, *rest = selective_scan_fn(u[..., chunk_start:chunk_end],
298
- delta[..., chunk_start:chunk_end],
299
- A,
300
- _B,
301
- _C,
302
- D,
303
- z=_z,
304
- delta_bias=delta_bias,
305
- delta_softplus=delta_softplus,
306
- return_last_state=return_last_state,
307
- prev_state=state if c > 0 else None)
299
+ out = selective_scan_fn(
300
+ u[..., chunk_start:chunk_end],
301
+ state,
302
+ delta[..., chunk_start:chunk_end],
303
+ A,
304
+ _B,
305
+ _C,
306
+ D,
307
+ z=_z,
308
+ delta_bias=delta_bias,
309
+ delta_softplus=delta_softplus,
310
+ has_initial_state=torch.ones(batch_size,
311
+ device=u.device,
312
+ dtype=torch.bool) if c > 0 else None)
313
outs.append(out)
309
- if return_last_state:
310
- state = rest[0]
314
if len(outs) > 1:
315
out = torch.cat(outs, dim=-1)
313
- out_ref, *rest = selective_scan_ref(u,
314
- delta,
315
- A,
316
- B,
317
- C,
318
- D,
319
- z=z,
320
- delta_bias=delta_bias,
321
- delta_softplus=delta_softplus,
322
- return_last_state=return_last_state)
323
- if return_last_state:
324
- state_ref = rest[0]
316
+
317
+ out_ref, state_ref, *rest = selective_scan_ref(
318
+ u_ref,
319
+ delta_ref,
320
+ A_ref,
321
+ B_ref,
322
+ C_ref,
323
+ D_ref,
324
+ z=z_ref,
325
+ delta_bias=delta_bias,
326
+ delta_softplus=delta_softplus,
327
+ return_last_state=True)
328
329
assert out is not None and out_ref is not None
330
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
328
- if return_last_state:
329
- assert state is not None and state_ref is not None
330
- assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
331
+ assert state is not None and state_ref is not None
332
+ assert torch.allclose(state, state_ref.to(itype), rtol=rtol, atol=atol)
333
334
selective_scan_opcheck_fn(u,
335
delta,
@@ -335,10 +337,10 @@ def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
337
B,
338
C,
339
D,
338
- z=z,
340
+ z,
341
delta_bias=delta_bias,
342
delta_softplus=delta_softplus,
341
- return_last_state=return_last_state)
343
+ ssm_states=state)
344
345
346
@pytest.mark.parametrize("itype",
@@ -391,9 +393,131 @@ def test_selective_state_update(dim, dstate, has_z, itype):
393
assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
394
395
396
+@pytest.mark.parametrize('wtype', [torch.float32])
397
+@pytest.mark.parametrize('itype', [torch.float32])
398
+@pytest.mark.parametrize('seqlen', [1, 128, 129, 256, 512, 1024, 2048, 4096])
399
+@pytest.mark.parametrize("return_last_state", [True])
400
+@pytest.mark.parametrize('has_delta_bias', [True])
401
+@pytest.mark.parametrize('delta_softplus', [True])
402
+@pytest.mark.parametrize('has_z', [True])
403
+@pytest.mark.parametrize('has_D', [True])
404
+@pytest.mark.parametrize("varBC_groups", [1, 2])
405
+@pytest.mark.parametrize("is_variable_C", [True])
406
+@pytest.mark.parametrize("is_variable_B", [True])
407
+def test_selective_scan_varlen(is_variable_B, is_variable_C, varBC_groups,
408
+ has_D, has_z, has_delta_bias, delta_softplus,
409
+ return_last_state, seqlen, itype, wtype):
410
+ if varBC_groups > 1 and (not is_variable_B or not is_variable_C):
411
+ pytest.skip() # This config is not applicable
412
+ device = 'cuda'
413
+ rtol, atol = (6e-4, 2e-3) if itype == torch.float32 else (3e-3, 5e-3)
414
+ if itype == torch.bfloat16:
415
+ rtol, atol = 3e-2, 5e-2
416
+ rtolw, atolw = (1e-3, 1e-3)
417
+ if has_z: # If we have z, the errors on the weights seem higher
418
+ rtolw = max(rtolw, rtol)
419
+ atolw = max(atolw, atol)
420
+ # set seed
421
+ torch.random.manual_seed(0)
422
+ seqlens = []
423
+ nsplits = 3
424
+ if seqlen < 10:
425
+ nsplits = 0
426
+ eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values
427
+ seqlens.append(
428
+ torch.diff(
429
+ torch.cat(
430
+ [torch.tensor([-1]), eos_pos,
431
+ torch.tensor([seqlen - 1])])).tolist())
432
+ assert sum(seqlens[-1]) == seqlen
433
+ assert all(s > 0 for s in seqlens[-1])
434
+
435
+ cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32)
436
+ cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum],
437
+ dim=0).cuda()
438
+
439
+ dim = 4
440
+ dstate = 8
441
+ A = (-0.5 * torch.rand(dim, dstate, device=device, dtype=wtype))
442
+ A_ref = A.clone()
443
+ B_shape = [varBC_groups, dstate, seqlen]
444
+ B = torch.randn(B_shape,
445
+ device=device,
446
+ dtype=wtype if not is_variable_B else itype)
447
+ B_ref = B.clone()
448
+ C_shape = [varBC_groups, dstate, seqlen]
449
+ C = torch.randn(C_shape,
450
+ device=device,
451
+ dtype=wtype if not is_variable_C else itype)
452
+ C_ref = C.clone()
453
+ D = torch.randn(dim, device=device, dtype=torch.float32) if has_D else None
454
+ D_ref = D.clone()
455
+ z = torch.randn(dim, seqlen, device=device, dtype=itype)
456
+ z_ref = z.clone()
457
+ delta_bias = (0.5 * torch.rand(dim, device=device, dtype=torch.float32)
458
+ ) if has_delta_bias else None
459
+ u = torch.randn(dim, seqlen, device=device, dtype=itype)
460
+ u_ref = u.clone()
461
+ delta = (0.5 * torch.rand(dim, seqlen, device=device, dtype=itype))
462
+ delta_ref = delta.clone()
463
+ out = None
464
+ out_ref = None
465
+ prev_state_shape = (cumsum.shape[0] - 1, u.shape[0], int(A.shape[1]))
466
+ prev_state = torch.randn(prev_state_shape,
467
+ device=u.device,
468
+ dtype=itype,
469
+ requires_grad=False)
470
+ prev_state_ref = prev_state.clone()
471
+ cache_indices = torch.randperm(cumsum.shape[0] - 1,
472
+ dtype=torch.int32,
473
+ device=u.device)
474
+
475
+ has_initial_state = torch.randint(0,
476
+ 2, (cumsum.shape[0] - 1, ),
477
+ dtype=torch.bool,
478
+ device=u.device)
479
+ out = selective_scan_fn(u, prev_state, delta, A, B, C, D, z, delta_bias,
480
+ delta_softplus, cumsum, cache_indices,
481
+ has_initial_state)
482
+ outs_ref = []
483
+ splits = [
484
+ torch.split(var, seqlens[0], dim=-1)
485
+ for var in (u_ref, delta_ref, B_ref, C_ref, z_ref)
486
+ ]
487
+ for i in range(len(seqlens[0])):
488
+ u_s, delta_s, B_s, C_s, z_s = [v[i].unsqueeze(0) for v in splits]
489
+ out_ref_s, _ = selective_scan_ref(
490
+ u_s,
491
+ delta_s,
492
+ A_ref,
493
+ B_s,
494
+ C_s,
495
+ D_ref,
496
+ z=z_s,
497
+ delta_bias=delta_bias,
498
+ delta_softplus=delta_softplus,
499
+ return_last_state=return_last_state,
500
+ prev_state=prev_state_ref[cache_indices[i]].unsqueeze(0)
501
+ if has_initial_state[i] else None,
502
+ final_state_out=prev_state_ref[cache_indices[i]].unsqueeze(0))
503
+ outs_ref.append(out_ref_s)
504
+ out_ref = torch.cat(outs_ref, dim=-1) if len(outs_ref) > 1 else outs_ref[0]
505
+
506
+ print("Output diff max", (out - out_ref[0]).max())
507
+ print("Output diff mean", (out - out_ref[0]).mean())
508
+ print("Output state diff max", (prev_state - prev_state_ref).max())
509
+ print("Output state diff mean", (prev_state - prev_state_ref).mean())
510
+ assert torch.allclose(prev_state, prev_state_ref, rtol=rtol, atol=atol)
511
+ assert torch.allclose(out, out_ref[0], rtol=rtol, atol=atol)
512
+
513
+ selective_scan_opcheck_fn(u, delta, A, B, C, D, z, delta_bias,
514
+ delta_softplus, cumsum, cache_indices,
515
+ has_initial_state, prev_state)
516
+
517
+
518
@pytest.mark.parametrize("itype",
519
[torch.float32, torch.float16, torch.bfloat16])
396
-@pytest.mark.parametrize("has_z", [False, True])
520
+@pytest.mark.parametrize("has_z", [True])
521
@pytest.mark.parametrize("dstate", [16, 32, 64])
522
@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
523
def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
@@ -405,7 +529,7 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
529
atol *= 2
530
# set seed
531
torch.random.manual_seed(0)
408
- batch_size = 16
532
+ batch_size = 3
533
534
total_entries = 10 * batch_size
535
state = torch.randn(total_entries, dim, dstate, dtype=itype, device=device)
@@ -443,6 +567,11 @@ def test_selective_state_update_with_batch_indices(dim, dstate, has_z, itype):
567
dt_bias=dt_bias,
568
dt_softplus=True)
569
570
+ print("Output diff max", (out - out_ref[0]).max())
571
+ print("Output diff mean", (out - out_ref[0]).mean())
572
+ print("Output state diff max", (state[state_indices, :] - state_ref).max())
573
+ print("Output state diff mean",
574
+ (state[state_indices, :] - state_ref).mean())
575
assert torch.allclose(state[state_indices, :],
576
state_ref,
577
rtol=rtol,
@@ -465,7 +594,7 @@ def test_selective_state_update_with_heads_with_batch_indices(
594
rtol, atol = 1e-1, 1e-1
595
# set seed
596
torch.random.manual_seed(0)
468
- batch_size = 16
597
+ batch_size = 3
598
headdim = 64
599
nheads = dim // headdim
600
tests/models/decoder_only/language/test_jamba.py
+115
-9
@@ -1,18 +1,16 @@
1
import pytest
2
3
+from vllm.sampling_params import SamplingParams
4
from vllm.worker.model_runner import _get_graph_batch_size
5
6
from ...utils import check_outputs_equal
7
7
-MODELS = ["ai21labs/Jamba-tiny-random"]
8
+MODELS = ["ai21labs/Jamba-tiny-dev"]
9
10
10
-# Fails due to usage of MoE as MLP(E=1_, which is different than the HF impl
11
-# TODO: Fix this with trained model
12
-@pytest.mark.skip()
11
@pytest.mark.parametrize("model", MODELS)
14
-@pytest.mark.parametrize("dtype", ["bfloat16"])
15
-@pytest.mark.parametrize("max_tokens", [10])
12
+@pytest.mark.parametrize("dtype", ["float"])
13
+@pytest.mark.parametrize("max_tokens", [96])
14
def test_models(
15
hf_runner,
16
vllm_runner,
@@ -22,7 +20,14 @@ def test_models(
20
max_tokens: int,
21
) -> None:
22
25
- with hf_runner(model, dtype=dtype) as hf_model:
23
+ with hf_runner(
24
+ model,
25
+ dtype=dtype,
26
+ model_kwargs={
27
+ "use_mamba_kernels":
28
+ False, # mamba kernels are not installed so HF
29
+ # don't use them
30
+ }) as hf_model:
31
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
32
33
with vllm_runner(model, dtype=dtype) as vllm_model:
@@ -38,8 +43,8 @@ def test_models(
43
44
45
@pytest.mark.parametrize("model", MODELS)
41
-@pytest.mark.parametrize("dtype", ["half"])
42
-@pytest.mark.parametrize("max_tokens", [5])
46
+@pytest.mark.parametrize("dtype", ["float"])
47
+@pytest.mark.parametrize("max_tokens", [96])
48
def test_batching(
49
vllm_runner,
50
example_prompts,
@@ -65,6 +70,107 @@ def test_batching(
70
)
71
72
73
+@pytest.mark.parametrize("model", MODELS)
74
+@pytest.mark.parametrize("dtype", ["float16"])
75
+@pytest.mark.parametrize("max_tokens", [10])
76
+def test_mamba_prefill_chunking_with_parallel_sampling(
77
+ hf_runner, vllm_runner, example_prompts, model: str, dtype: str,
78
+ max_tokens: int) -> None:
79
+ # Tests prefill chunking in conjunction with n>1, in this case,
80
+ # prefill is populated with decoding tokens and we test that it
81
+ # doesn't fail This test might fail if cache is not allocated
82
+ # correctly for n > 1 decoding steps inside a
83
+ # chunked prefill forward pass (where we have both prefills
84
+ # and decoding together )
85
+ sampling_params = SamplingParams(n=3,
86
+ temperature=1,
87
+ seed=0,
88
+ max_tokens=max_tokens)
89
+ with vllm_runner(
90
+ model,
91
+ dtype=dtype,
92
+ enable_chunked_prefill=True,
93
+ max_num_batched_tokens=30,
94
+ max_num_seqs=10 # forces prefill chunks with decoding
95
+ ) as vllm_model:
96
+ vllm_model.generate(example_prompts, sampling_params)
97
+
98
+
99
+@pytest.mark.parametrize("model", MODELS)
100
+@pytest.mark.parametrize("dtype", ["bfloat16"])
101
+@pytest.mark.parametrize("max_tokens", [10])
102
+def test_mamba_prefill_chunking(hf_runner, vllm_runner, example_prompts,
103
+ model: str, dtype: str,
104
+ max_tokens: int) -> None:
105
+ # numeric error during prefill chucking produces different generation
106
+ # compared to w/o prefill chunking for those examples, removed them for now
107
+ example_prompts.pop(7)
108
+ example_prompts.pop(2)
109
+ example_prompts.pop(1)
110
+
111
+ with hf_runner(
112
+ model,
113
+ dtype=dtype,
114
+ model_kwargs={
115
+ "use_mamba_kernels":
116
+ False, # mamba kernels are not installed so HF
117
+ # don't use them
118
+ }) as hf_model:
119
+ non_chunked = hf_model.generate_greedy(example_prompts, max_tokens)
120
+
121
+ with vllm_runner(model,
122
+ dtype=dtype,
123
+ enable_chunked_prefill=True,
124
+ max_num_batched_tokens=5,
125
+ max_num_seqs=2) as vllm_model:
126
+ chunked = vllm_model.generate_greedy(example_prompts,
127
+ max_tokens=max_tokens)
128
+
129
+ check_outputs_equal(
130
+ outputs_0_lst=chunked,
131
+ outputs_1_lst=non_chunked,
132
+ name_0="chunked",
133
+ name_1="non_chunked",
134
+ )
135
+
136
+
137
+@pytest.mark.parametrize("model", MODELS)
138
+@pytest.mark.parametrize("dtype", ["bfloat16"])
139
+@pytest.mark.parametrize("max_tokens", [15])
140
+def test_parallel_sampling(
141
+ vllm_runner,
142
+ example_prompts,
143
+ model: str,
144
+ dtype: str,
145
+ max_tokens: int,
146
+) -> None:
147
+
148
+ with vllm_runner(model, dtype=dtype) as vllm_model:
149
+ for_loop_outputs = []
150
+ for _ in range(10):
151
+ for_loop_outputs.append(
152
+ # using example_prompts index 1 instead of 0 since with 0 the
153
+ # logprobs get really close and the test doesn't pass
154
+ vllm_model.generate_greedy([example_prompts[1]], max_tokens)
155
+ [0])
156
+ sampling_params = SamplingParams(n=10,
157
+ temperature=0.001,
158
+ seed=0,
159
+ max_tokens=max_tokens)
160
+ n_lt_1_outputs = vllm_model.generate([example_prompts[1]],
161
+ sampling_params)
162
+ token_ids, texts = n_lt_1_outputs[0]
163
+ n_lt_1_outputs = [(token_id, text)
164
+ for token_id, text in zip(token_ids, texts)]
165
+
166
+ check_outputs_equal(
167
+ outputs_0_lst=n_lt_1_outputs,
168
+ outputs_1_lst=for_loop_outputs,
169
+ name_0="vllm_n_lt_1_outputs",
170
+ name_1="vllm",
171
+ )
172
+
173
+
174
@pytest.mark.parametrize("model", MODELS)
175
@pytest.mark.parametrize("dtype", ["bfloat16"])
176
@pytest.mark.parametrize("max_tokens", [20])
vllm/_custom_ops.py
+39
-38
@@ -440,9 +440,10 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
440
@torch.library.register_fake("_C::causal_conv1d_fwd")
441
def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
442
bias_: Optional[torch.Tensor],
443
- seq_idx_: Optional[torch.Tensor],
444
- initial_states_: Optional[torch.Tensor],
445
- final_states_out_: Optional[torch.Tensor],
443
+ conv_states: Optional[torch.Tensor],
444
+ cu_seq_len: Optional[torch.Tensor],
445
+ cache_indices: Optional[torch.Tensor],
446
+ has_initial_state: Optional[torch.Tensor],
447
silu_activation: bool) -> torch.Tensor:
448
return torch.empty_like(x)
449
@@ -450,22 +451,22 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
451
def causal_conv1d_update_fake(
452
x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
453
bias_: Optional[torch.Tensor], silu_activation: bool,
454
+ cache_seqlens: Optional[torch.Tensor],
455
conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
456
return torch.empty_like(x)
457
458
@torch.library.register_fake("_C::selective_scan_fwd")
457
- def selective_scan_fwd_fake(
458
- u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
459
- B: torch.Tensor, C: torch.Tensor, D_: Optional[torch.Tensor],
460
- z_: Optional[torch.Tensor], delta_bias_: Optional[torch.Tensor],
461
- delta_softplus: bool, index_: Optional[torch.Tensor],
462
- x: Optional[torch.Tensor]) -> List[torch.Tensor]:
463
- a = torch.empty_like(u)
464
- if z_ is not None:
465
- c = torch.empty_like(z_)
466
- return [a, c]
467
- else:
468
- return [a]
459
+ def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor,
460
+ A: torch.Tensor, B: torch.Tensor,
461
+ C: torch.Tensor, D_: Optional[torch.Tensor],
462
+ z_: Optional[torch.Tensor],
463
+ delta_bias_: Optional[torch.Tensor],
464
+ delta_softplus: bool,
465
+ cu_seq_len: Optional[torch.Tensor],
466
+ cache_indices: Optional[torch.Tensor],
467
+ has_initial_state: Optional[torch.Tensor],
468
+ ssm_states: Optional[torch.Tensor]) -> None:
469
+ return None
470
471
472
# cutlass
@@ -761,37 +762,37 @@ def ggml_mul_mat_a8(
762
# mamba
763
def causal_conv1d_fwd(x: torch.Tensor, weight: torch.Tensor,
764
bias_: Optional[torch.Tensor],
764
- seq_idx_: Optional[torch.Tensor],
765
- initial_states_: Optional[torch.Tensor],
766
- final_states_out_: Optional[torch.Tensor],
765
+ conv_states: Optional[torch.Tensor],
766
+ query_start_loc: Optional[torch.Tensor],
767
+ cache_indices: Optional[torch.Tensor],
768
+ has_initial_state: Optional[torch.Tensor],
769
silu_activation: bool) -> torch.Tensor:
768
- return torch.ops._C.causal_conv1d_fwd(x, weight, bias_, seq_idx_,
769
- initial_states_, final_states_out_,
770
- silu_activation)
770
+ return torch.ops._C.causal_conv1d_fwd(x, weight, bias_, conv_states,
771
+ query_start_loc, cache_indices,
772
+ has_initial_state, silu_activation)
773
774
775
def causal_conv1d_update(
774
- x: torch.Tensor,
775
- conv_state: torch.Tensor,
776
- weight: torch.Tensor,
777
- bias_: Optional[torch.Tensor],
778
- silu_activation: bool,
779
- conv_state_indices: Optional[torch.Tensor],
780
-) -> torch.Tensor:
776
+ x: torch.Tensor, conv_state: torch.Tensor, weight: torch.Tensor,
777
+ bias_: Optional[torch.Tensor], silu_activation: bool,
778
+ cache_seqlens: Optional[torch.Tensor],
779
+ conv_state_indices: Optional[torch.Tensor]) -> torch.Tensor:
780
return torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
782
- silu_activation,
781
+ silu_activation, cache_seqlens,
782
conv_state_indices)
783
784
786
-def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
787
- B: torch.Tensor, C: torch.Tensor,
788
- D_: Optional[torch.Tensor], z_: Optional[torch.Tensor],
789
- delta_bias_: Optional[torch.Tensor],
790
- delta_softplus: bool, index_: Optional[torch.Tensor],
791
- x: Optional[torch.Tensor]) -> List[torch.Tensor]:
792
- return torch.ops._C.selective_scan_fwd(u, delta, A, B, C, D_, z_,
793
- delta_bias_, delta_softplus, index_,
794
- x)
785
+def selective_scan_fwd(
786
+ u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor, B: torch.Tensor,
787
+ C: torch.Tensor, D_: Optional[torch.Tensor],
788
+ z_: Optional[torch.Tensor], delta_bias_: Optional[torch.Tensor],
789
+ delta_softplus: bool, query_start_loc: Optional[torch.Tensor],
790
+ cache_indices: Optional[torch.Tensor],
791
+ has_initial_state: Optional[torch.Tensor], ssm_states: torch.Tensor):
792
+ torch.ops._C.selective_scan_fwd(u, delta, A, B, C, D_, z_, delta_bias_,
793
+ delta_softplus, query_start_loc,
794
+ cache_indices, has_initial_state,
795
+ ssm_states)
796
797
798
# moe
vllm/model_executor/layers/mamba/ops/causal_conv1d.py
+42
-45
@@ -12,59 +12,44 @@ def causal_conv1d_fn(
12
x: torch.Tensor,
13
weight: torch.Tensor,
14
bias: Optional[torch.Tensor] = None,
15
- seq_idx: Optional[torch.Tensor] = None,
16
- initial_states: Optional[torch.Tensor] = None,
17
- return_final_states: bool = False,
18
- final_states_out=None,
19
- activation: str = "silu",
15
+ query_start_loc: Optional[torch.Tensor] = None,
16
+ cache_indices: Optional[torch.Tensor] = None,
17
+ has_initial_state: Optional[torch.Tensor] = None,
18
+ conv_states: Optional[torch.Tensor] = None,
19
+ activation: Optional[str] = "silu",
20
):
21
"""
22
- x: (batch, dim, seqlen)
22
+ x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen
23
+ sequences are concatenated from left to right for varlen
24
weight: (dim, width)
25
bias: (dim,)
25
- seq_idx: (batch, seqlen)
26
- initial_states: (batch, dim, width - 1)
27
- final_states_out: (batch, dim, width - 1), to be written to
26
+ query_start_loc: (batch + 1) int32
27
+ The cumulative sequence lengths of the sequences in
28
+ the batch, used to index into sequence. prepended by 0.
29
+ for example: query_start_loc = torch.Tensor([0,10,16,17]),
30
+ x.shape=(dim,17)
31
+ cache_indices: (batch) int32
32
+ indicates the corresponding state index,
33
+ like so: conv_state = conv_states[cache_indices[batch_id]]
34
+ has_initial_state: (batch) bool
35
+ indicates whether should the kernel take the current state as initial
36
+ state for the calculations
37
+ conv_states: (...,dim,width - 1) itype
38
+ updated inplace if provided
39
activation: either None or "silu" or "swish"
40
41
out: (batch, dim, seqlen)
42
"""
43
if activation not in [None, "silu", "swish"]:
44
raise NotImplementedError("activation must be None, silu, or swish")
34
- if x.stride(2) != 1 and x.stride(1) != 1:
45
+ if x.stride(-1) != 1:
46
x = x.contiguous()
47
bias = bias.contiguous() if bias is not None else None
37
- if seq_idx is not None:
38
- assert (initial_states is
39
- None), "initial_states must be None if seq_idx is not None"
40
- assert (not return_final_states
41
- ), "If seq_idx is not None, we don't return final_states_out"
42
- seq_idx = seq_idx.contiguous() if seq_idx is not None else None
43
- if initial_states is not None and (initial_states.stride(2) != 1
44
- and initial_states.stride(1) != 1):
45
- initial_states = initial_states.contiguous()
46
- if return_final_states:
47
- assert (
48
- x.stride(1) == 1
49
- ), "Only channel-last layout support returning final_states_out"
50
- if final_states_out is not None:
51
- assert (final_states_out.stride(2) == 1
52
- or final_states_out.stride(1) == 1)
53
- else:
54
- batch, dim, seqlen = x.shape
55
- width = weight.shape[1]
56
- final_states_out = torch.empty(batch,
57
- width - 1,
58
- dim,
59
- device=x.device,
60
- dtype=x.dtype).transpose(1, 2)
61
- else:
62
- final_states_out = None
48
64
- out = ops.causal_conv1d_fwd(x, weight, bias, seq_idx, initial_states,
65
- final_states_out, activation
49
+ out = ops.causal_conv1d_fwd(x, weight, bias, conv_states, query_start_loc,
50
+ cache_indices, has_initial_state, activation
51
in ["silu", "swish"])
67
- return (out, None) if not return_final_states else (out, final_states_out)
52
+ return out
53
54
55
def causal_conv1d_update(x: torch.Tensor,
@@ -72,21 +57,33 @@ def causal_conv1d_update(x: torch.Tensor,
57
weight: torch.Tensor,
58
bias: Optional[torch.Tensor] = None,
59
activation: Optional[str] = None,
60
+ cache_seqlens: Optional[torch.Tensor] = None,
61
conv_state_indices: Optional[torch.Tensor] = None):
62
"""
77
- x: (batch, dim)
78
- conv_state: (batch, dim, width)
63
+ x: (batch, dim) or (batch, dim, seqlen)
64
+ conv_state: (batch, dim, state_len), where state_len >= width - 1
65
weight: (dim, width)
66
bias: (dim,)
67
+ cache_seqlens: (batch,), dtype int32.
68
+ If not None, the conv_state is treated as a circular buffer.
69
+ The conv_state will be updated by copying x to the conv_state
70
+ starting at the index
71
+ @cache_seqlens % state_len.
72
conv_state_indices: (batch,), dtype int32
73
If not None, the conv_state is a larger tensor along the batch dim,
74
and we are selecting the batch coords specified by conv_state_indices.
75
Useful for a continuous batching scenario.
76
86
- out: (batch, dim)
77
+ out: (batch, dim) or (batch, dim, seqlen)
78
"""
79
if activation not in [None, "silu", "swish"]:
80
raise NotImplementedError("activation must be None, silu, or swish")
90
- activation_bool = activation in ["silu", "swish"]
91
- return ops.causal_conv1d_update(x, conv_state, weight, bias,
92
- activation_bool, conv_state_indices)
81
+ activation_val = activation in ["silu", "swish"]
82
+ unsqueeze = x.dim() == 2
83
+ if unsqueeze:
84
+ x = x.unsqueeze(-1)
85
+ out = ops.causal_conv1d_update(x, conv_state, weight, bias, activation_val,
86
+ cache_seqlens, conv_state_indices)
87
+ if unsqueeze:
88
+ out = out.squeeze(-1)
89
+ return out
vllm/model_executor/layers/mamba/ops/mamba_ssm.py
+59
-35
@@ -1,6 +1,8 @@
1
# Copyright (c) 2024, Tri Dao, Albert Gu.
2
# Adapted from https://github.com/state-spaces/mamba/blob/main/mamba_ssm/ops/triton/selective_state_update.py
3
4
+from typing import Tuple
5
+
6
import torch
7
import triton
8
import triton.language as tl
@@ -317,20 +319,50 @@ def selective_state_update(state,
319
return out
320
321
320
-def selective_scan_fn(u,
321
- delta,
322
- A,
323
- B,
324
- C,
325
- D=None,
326
- z=None,
327
- delta_bias=None,
328
- delta_softplus=False,
329
- return_last_state=False,
330
- position_indices=None,
331
- prev_state=None):
332
- """if return_last_state is True, returns (out, last_state)
333
- last_state has shape (batch, dim, dstate).
322
+def selective_scan_fn(
323
+ u,
324
+ ssm_states,
325
+ delta,
326
+ A,
327
+ B,
328
+ C,
329
+ D=None,
330
+ z=None,
331
+ delta_bias=None,
332
+ delta_softplus=False,
333
+ query_start_loc=None,
334
+ cache_indices=None,
335
+ has_initial_state=None) -> Tuple[torch.Tensor, torch.Tensor]:
336
+ """
337
+ u: (dim, total_length) for varlen or (batch, dim, seqlen)
338
+ delta: (dim, total_length) for varlen or (batch, dim, seqlen)
339
+ A: (dim, dstate)
340
+ B: (ngroups, dstate, total_length) for varlen or
341
+ (batch,ngroups,dstate,seqlen)
342
+ C: (ngroups, dstate, total_length) for varlen or
343
+ (batch,ngroups,dstate,seqlen)
344
+ D: (dim,)
345
+ z: (dim, total_length) for varlen or (batch, dim, seqlen)
346
+ dt_bias: (dim,) or (dim)
347
+ query_start_loc: (batch + 1) int32
348
+ The cumulative sequence lengths of the sequences in
349
+ the batch, used to index into sequence. prepended with 0.
350
+ for example: query_start_loc = torch.Tensor([0,10,16,17]),
351
+ x.shape=(dim,17)
352
+ cache_indices: (batch) int32
353
+ A tensor with each cell is a correspondent
354
+ input and output ssm_state index
355
+ has_initial_state: (batch) bool
356
+ A tensor populated with ones and zeros,
357
+ indicate if the ssm_state at the corresponding index should be
358
+ used as initial state. Not providing argument assumes
359
+ there's no initial state
360
+
361
+ returns
362
+ output: (dim, total_length) for varlen or (batch, dim, seqlen)
363
+ supports inplace replacement
364
+ last_state has shape (batch, dim, dstate).
365
+ supports inplace replacement if ssm_state was provided
366
"""
367
if u.stride(-1) != 1:
368
u = u.contiguous()
@@ -344,28 +376,20 @@ def selective_scan_fn(u,
376
C = C.contiguous()
377
if z is not None and z.stride(-1) != 1:
378
z = z.contiguous()
347
- if B.dim() == 3:
379
+ if B.dim() == 3 and query_start_loc is None:
380
B = B.unsqueeze(1)
349
- if C.dim() == 3:
381
+ if B.dim() == 2 and query_start_loc is not None:
382
+ B = B.unsqueeze(0)
383
+ if C.dim() == 3 and query_start_loc is None:
384
C = C.unsqueeze(1)
351
- n_chunks = int((u.shape[-1] + 2048 - 1) / 2048)
352
- x = torch.zeros((
353
- u.shape[0],
354
- u.shape[1],
355
- n_chunks,
356
- int(A.shape[1] * 2),
357
- ),
358
- device=u.device,
359
- dtype=torch.float32,
360
- requires_grad=False)
361
- x[:, :, 0, 0::2] = 1
362
- if prev_state is not None:
363
- x[:, :, 0, 1::2].copy_(prev_state)
364
- out, *rest = ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias,
365
- delta_softplus, position_indices, x)
366
- last_state = x[:, :, -1, 1::2] # (batch, dim, dstate)
385
+ if C.dim() == 2 and query_start_loc is not None:
386
+ C = C.unsqueeze(0)
387
+
388
+ ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias, delta_softplus,
389
+ query_start_loc, cache_indices, has_initial_state,
390
+ ssm_states)
391
+
392
if z is None:
368
- return out if not return_last_state else (out, last_state)
393
+ return delta # output written inplace to delta
394
else:
370
- out_z = rest[0]
371
- return out_z if not return_last_state else (out_z, last_state)
395
+ return z # output written inplace to z
vllm/model_executor/models/jamba.py
+67
-97
@@ -138,42 +138,47 @@ class JambaMambaMixer(nn.Module):
138
self.c_layernorm = RMSNorm(self.ssm_state_size,
139
eps=config.rms_norm_eps)
140
141
- def mamba_forward(self,
142
- hidden_states: torch.Tensor,
143
- cache_params: MambaCacheParams = None):
141
+ def forward(self, hidden_states: torch.Tensor,
142
+ attn_metadata: AttentionMetadata, conv_state: torch.Tensor,
143
+ ssm_state: torch.Tensor):
144
+
145
# 1. Gated MLP's linear projection
145
- projected_states = self.in_proj(hidden_states)[0].transpose(1, 2)
146
- hidden_states, gate = projected_states.chunk(2, dim=1)
146
+ projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
147
+ hidden_states, gate = projected_states.chunk(2, dim=-2)
148
149
# 2. Convolution sequence transformation
150
conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
151
self.conv1d.weight.size(2))
151
- if cache_params is not None and not cache_params.is_prompt:
152
- hidden_states = causal_conv1d_update(
153
- hidden_states.squeeze(-1),
154
- cache_params.conv_state,
155
- conv_weights,
156
- self.conv1d.bias,
157
- self.activation,
158
- )
159
- hidden_states = hidden_states.unsqueeze(-1)
160
- else:
161
- if cache_params is not None:
162
- conv_states = nn.functional.pad(
163
- hidden_states,
164
- (self.conv_kernel_size - hidden_states.shape[-1], 0))
165
- cache_params.conv_state.copy_(conv_states)
152
167
- hidden_states, _ = causal_conv1d_fn(
153
+ if attn_metadata.query_start_loc is not None \
154
+ and attn_metadata.context_lens_tensor is not None:
155
+ # |---------- N-1 iteration --------|
156
+ # |---------------- N iteration ---------------------|
157
+ # |- tokenA -|......................|-- newTokens ---|
158
+ # |---------- context_len ----------|
159
+ # |-------------------- seq_len ---------------------|
160
+ # |-- query_len ---|
161
+ hidden_states = causal_conv1d_fn(
162
hidden_states,
163
conv_weights,
164
self.conv1d.bias,
165
activation=self.activation,
166
+ conv_states=conv_state,
167
+ has_initial_state=attn_metadata.context_lens_tensor > 0,
168
+ query_start_loc=attn_metadata.query_start_loc)
169
+ else:
170
+ hidden_states = causal_conv1d_update(
171
+ hidden_states.transpose(0, 1),
172
+ conv_state,
173
+ conv_weights,
174
+ self.conv1d.bias,
175
+ self.activation,
176
)
177
+ hidden_states = hidden_states.transpose(0, 1)
178
179
# 3. State Space Model sequence transformation
180
# 3.a. input varying initialization of time_step, B and C
176
- ssm_parameters = self.x_proj(hidden_states.transpose(1, 2))[0]
181
+ ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
182
183
time_step, B, C = torch.split(
184
ssm_parameters,
@@ -184,72 +189,46 @@ class JambaMambaMixer(nn.Module):
189
B = self.b_layernorm(B.contiguous())
190
C = self.c_layernorm(C.contiguous())
191
187
- discrete_time_step = self.dt_proj(time_step)[0].transpose(1, 2)
192
+ discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
193
# 3.c perform the recurrence y ← SSM(A, B, C)(x)
194
time_proj_bias = (self.dt_proj.bias.float() if hasattr(
195
self.dt_proj, "bias") else None)
191
- if cache_params is not None and not cache_params.is_prompt:
192
- scan_outputs = selective_state_update(
193
- cache_params.ssm_state,
194
- hidden_states[..., 0],
195
- discrete_time_step[..., 0],
196
- self.A,
197
- B[:, 0],
198
- C[:, 0],
199
- self.D,
200
- gate[..., 0],
201
- time_proj_bias,
202
- dt_softplus=True,
203
- ).unsqueeze(-1)
204
- else:
205
- scan_outputs, ssm_state = selective_scan_fn(
196
+
197
+ if attn_metadata.query_start_loc is not None \
198
+ and attn_metadata.context_lens_tensor is not None:
199
+ scan_outputs = selective_scan_fn(
200
hidden_states,
201
+ ssm_state,
202
discrete_time_step,
203
self.A,
209
- B.transpose(1, 2),
210
- C.transpose(1, 2),
204
+ B.transpose(-2, -1),
205
+ C.transpose(-2, -1),
206
self.D.float(),
207
gate,
208
time_proj_bias,
209
delta_softplus=True,
215
- return_last_state=True,
210
+ has_initial_state=attn_metadata.context_lens_tensor > 0,
211
+ query_start_loc=attn_metadata.query_start_loc)
212
+ else:
213
+ scan_outputs = selective_state_update(
214
+ ssm_state,
215
+ hidden_states.transpose(0, 1),
216
+ discrete_time_step.transpose(0, 1),
217
+ self.A,
218
+ B,
219
+ C,
220
+ self.D,
221
+ gate.transpose(0, 1),
222
+ time_proj_bias,
223
+ dt_softplus=True,
224
)
217
- if ssm_state is not None and cache_params is not None:
218
- cache_params.ssm_state.copy_(ssm_state)
225
+ scan_outputs = scan_outputs.transpose(0, 1)
226
227
# 4. Final linear projection
221
- contextualized_states = self.out_proj(scan_outputs.transpose(1, 2))[0]
228
+ contextualized_states = self.out_proj(scan_outputs.transpose(-2,
229
+ -1))[0]
230
return contextualized_states
231
224
- def forward(
225
- self,
226
- hidden_states: torch.Tensor,
227
- attn_metadata: AttentionMetadata,
228
- conv_state: torch.Tensor,
229
- ssm_state: torch.Tensor,
230
- ):
231
- if attn_metadata.prefill_metadata is not None:
232
- offset = 0
233
- for i, prompt_len in enumerate(
234
- attn_metadata.prefill_metadata.seq_lens):
235
- cache = MambaCacheParams(True,
236
- conv_state=conv_state[i].unsqueeze(0),
237
- ssm_state=ssm_state[i].unsqueeze(0))
238
- hidden_states[offset:offset + prompt_len].copy_(
239
- self.mamba_forward(hidden_states[offset:offset +
240
- prompt_len].unsqueeze(0),
241
- cache_params=cache)[0])
242
- offset += prompt_len
243
- else:
244
- cache = MambaCacheParams(False,
245
- conv_state=conv_state,
246
- ssm_state=ssm_state)
247
- hidden_states = self.mamba_forward(hidden_states.unsqueeze(1),
248
- cache_params=cache)
249
- hidden_states = hidden_states.squeeze(1)
250
-
251
- return hidden_states
252
-
232
233
class JambaMoE(nn.Module):
234
@@ -571,8 +550,6 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
550
lora_config: Optional[LoRAConfig] = None,
551
scheduler_config: Optional[SchedulerConfig] = None,
552
) -> None:
574
- assert not scheduler_config.chunked_prefill_enabled, \
575
- "Jamba currently does not support chunked prefill"
553
assert not cache_config.enable_prefix_caching, \
554
"Jamba currently does not support prefix caching"
555
@@ -616,18 +593,10 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
593
594
if "seqlen_agnostic_capture_inputs" not in kwargs:
595
# We get here only on Prefill/Eager mode runs
619
- assert all(
620
- key in kwargs
621
- for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
622
-
596
request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
597
finished_requests_ids = kwargs["finished_requests_ids"]
625
- self._release_mamba_cache(finished_requests_ids)
626
- batch_size = input_ids.shape[0]
627
- if attn_metadata.prefill_metadata:
628
- batch_size = len(request_ids_to_seq_ids)
629
- mamba_cache = self._prepare_current_run_mamba_cache(
630
- request_ids_to_seq_ids, batch_size, finished_requests_ids)
598
+ mamba_cache = self._release_finished_and_prepare_mamba_cache(
599
+ finished_requests_ids, request_ids_to_seq_ids)
600
else:
601
# CUDA graph capturing runs
602
mamba_cache = kwargs["seqlen_agnostic_capture_inputs"]
@@ -699,13 +668,15 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
668
669
def _prepare_current_run_mamba_cache(
670
self, request_ids_to_seq_ids: Dict[str, list[int]],
702
- batch_size: int, finished_requests_ids: List[str]):
671
+ finished_requests_ids: List[str]
672
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
673
running_indices = []
674
request_ids_to_seq_ids_flatten = [
675
(req_id, seq_id)
676
for req_id, seq_ids in request_ids_to_seq_ids.items()
677
for seq_id in seq_ids
678
]
679
+ batch_size = len(request_ids_to_seq_ids_flatten)
680
for dest_index, (request_id,
681
seq_id) in enumerate(request_ids_to_seq_ids_flatten):
682
if request_id in finished_requests_ids:
@@ -769,22 +740,21 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
740
seq_ids2index.update({seq_id: to_index})
741
return
742
743
+ def _release_finished_and_prepare_mamba_cache(
744
+ self, finished_requests_ids,
745
+ request_ids_to_seq_ids) -> Tuple[torch.Tensor, torch.Tensor]:
746
+ self._release_mamba_cache(finished_requests_ids)
747
+ return self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
748
+ finished_requests_ids)
749
+
750
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
751
"""
752
Copy the relevant Mamba cache into the CUDA graph input buffer
753
that was provided during the capture runs
754
(JambaForCausalLM.mamba_gc_cache_buffer).
755
"""
778
- assert all(
779
- key in kwargs
780
- for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
781
- finished_requests_ids = kwargs["finished_requests_ids"]
782
- self._release_mamba_cache(finished_requests_ids)
783
- request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
784
- cg_batch_size = input_buffers['input_ids'].shape[0]
785
- self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
786
- cg_batch_size,
787
- finished_requests_ids)
756
+ self._release_finished_and_prepare_mamba_cache(
757
+ kwargs["finished_requests_ids"], kwargs["request_ids_to_seq_ids"])
758
759
def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
760
"""
@@ -819,7 +789,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
789
hidden_size = self.config.hidden_size
790
conv_state_shape = (
791
self.config.mamba_expand * hidden_size // world_size,
822
- self.config.mamba_d_conv,
792
+ self.config.mamba_d_conv - 1,
793
)
794
temporal_state_shape = (
795
self.config.mamba_expand * self.config.hidden_size // world_size,