A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Kernel/Model] Migrate mamba_ssm and causal_conv1d kernels to vLLM (#7651)
Mor Zusman committed
Aug 29, 2024 at 01:06 UTC
fdd9daafa3b31746ec8ec7c0d67ebc7efeb13f8f
20 files changed
+2815
-31
CMakeLists.txt
+2
@@ -203,6 +203,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
203
FetchContent_MakeAvailable(cutlass)
204
205
list(APPEND VLLM_EXT_SRC
206
+ "csrc/mamba/mamba_ssm/selective_scan_fwd.cu"
207
+ "csrc/mamba/causal_conv1d/causal_conv1d.cu"
208
"csrc/quantization/aqlm/gemm_kernels.cu"
209
"csrc/quantization/awq/gemm_kernels.cu"
210
"csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
Dockerfile
-23
@@ -42,9 +42,6 @@ COPY requirements-cuda.txt requirements-cuda.txt
42
RUN --mount=type=cache,target=/root/.cache/pip \
43
python3 -m pip install -r requirements-cuda.txt
44
45
-COPY requirements-mamba.txt requirements-mamba.txt
46
-RUN python3 -m pip install packaging
47
-RUN python3 -m pip install -r requirements-mamba.txt
45
46
# cuda arch list used by torch
47
# can be useful for both `dev` and `test`
@@ -127,22 +124,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
124
python3 -m pip install -r requirements-dev.txt
125
126
#################### DEV IMAGE ####################
130
-#################### MAMBA Build IMAGE ####################
131
-FROM dev as mamba-builder
132
-# max jobs used for build
133
-ARG max_jobs=2
134
-ENV MAX_JOBS=${max_jobs}
135
-
136
-WORKDIR /usr/src/mamba
137
-
138
-COPY requirements-mamba.txt requirements-mamba.txt
139
-
140
-# Download the wheel or build it if a pre-compiled release doesn't exist
141
-RUN pip --verbose wheel -r requirements-mamba.txt \
142
- --no-build-isolation --no-deps --no-cache-dir
143
-
144
-#################### MAMBA Build IMAGE ####################
145
-
127
#################### vLLM installation IMAGE ####################
128
# image with vLLM installed
129
FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu20.04 AS vllm-base
@@ -179,10 +160,6 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
160
--mount=type=cache,target=/root/.cache/pip \
161
python3 -m pip install dist/*.whl --verbose
162
182
-RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamba \
183
- --mount=type=cache,target=/root/.cache/pip \
184
- python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
185
-
163
RUN --mount=type=cache,target=/root/.cache/pip \
164
. /etc/environment && \
165
python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl
csrc/mamba/causal_conv1d/causal_conv1d.cu
new
+700
@@ -0,0 +1,700 @@
1
+// clang-format off
2
+// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_fwd.cu
3
+// and https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_update.cu
4
+#include <torch/all.h>
5
+#include <ATen/cuda/CUDAContext.h>
6
+#include <c10/cuda/CUDAGuard.h>
7
+
8
+#include "causal_conv1d.h"
9
+#include <c10/util/BFloat16.h>
10
+#include <c10/util/Half.h>
11
+#include <c10/cuda/CUDAException.h> // For C10_CUDA_CHECK and C10_CUDA_KERNEL_LAUNCH_CHECK
12
+
13
+#include <cub/block/block_load.cuh>
14
+#include <cub/block/block_store.cuh>
15
+
16
+#include "static_switch.h"
17
+
18
+
19
+
20
+#define CHECK_SHAPE(x, ...) TORCH_CHECK(x.sizes() == torch::IntArrayRef({__VA_ARGS__}), #x " must have shape (" #__VA_ARGS__ ")")
21
+
22
+#define DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(ITYPE, NAME, ...) \
23
+ if (ITYPE == at::ScalarType::Half) { \
24
+ using input_t = at::Half; \
25
+ using weight_t = at::Half; \
26
+ __VA_ARGS__(); \
27
+ } else if (ITYPE == at::ScalarType::BFloat16) { \
28
+ using input_t = at::BFloat16; \
29
+ using weight_t = at::BFloat16; \
30
+ __VA_ARGS__(); \
31
+ } else if (ITYPE == at::ScalarType::Float) { \
32
+ using input_t = float; \
33
+ using weight_t = float; \
34
+ __VA_ARGS__(); \
35
+ } else { \
36
+ AT_ERROR(#NAME, " not implemented for input type '", toString(ITYPE), "'"); \
37
+ }
38
+
39
+
40
+template<typename input_t, typename weight_t>
41
+void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream);
42
+template <typename input_t, typename weight_t>
43
+void causal_conv1d_channellast_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream);
44
+
45
+template<typename input_t, typename weight_t>
46
+void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream);
47
+
48
+void set_conv_params_fwd(ConvParamsBase ¶ms,
49
+ // sizes
50
+ const size_t batch,
51
+ const size_t dim,
52
+ const size_t seqlen,
53
+ const size_t width,
54
+ // device pointers
55
+ const at::Tensor x,
56
+ const at::Tensor weight,
57
+ const at::Tensor out,
58
+ void* bias_ptr,
59
+ bool silu_activation) {
60
+
61
+ // Reset the parameters
62
+ memset(¶ms, 0, sizeof(params));
63
+
64
+ params.batch = batch;
65
+ params.dim = dim;
66
+ params.seqlen = seqlen;
67
+ params.width = width;
68
+
69
+ params.silu_activation = silu_activation;
70
+
71
+ // Set the pointers and strides.
72
+ params.x_ptr = x.data_ptr();
73
+ params.weight_ptr = weight.data_ptr();
74
+ params.bias_ptr = bias_ptr;
75
+ params.out_ptr = out.data_ptr();
76
+ // All stride are in elements, not bytes.
77
+ params.x_batch_stride = x.stride(0);
78
+ params.x_c_stride = x.stride(1);
79
+ params.x_l_stride = x.stride(-1);
80
+ params.weight_c_stride = weight.stride(0);
81
+ params.weight_width_stride = weight.stride(1);
82
+ params.out_batch_stride = out.stride(0);
83
+ params.out_c_stride = out.stride(1);
84
+ params.out_l_stride = out.stride(-1);
85
+}
86
+
87
+
88
+at::Tensor
89
+causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
90
+ const c10::optional<at::Tensor> &bias_,
91
+ const c10::optional<at::Tensor> &seq_idx_,
92
+ const c10::optional<at::Tensor> &initial_states_,
93
+ const c10::optional<at::Tensor> &final_states_out_,
94
+ bool silu_activation) {
95
+ auto input_type = x.scalar_type();
96
+ auto weight_type = weight.scalar_type();
97
+ TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
98
+ TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16);
99
+
100
+ TORCH_CHECK(x.is_cuda());
101
+ TORCH_CHECK(weight.is_cuda());
102
+
103
+ const auto sizes = x.sizes();
104
+ const int batch_size = sizes[0];
105
+ const int dim = sizes[1];
106
+ const int seqlen = sizes[2];
107
+ const int width = weight.size(-1);
108
+
109
+ CHECK_SHAPE(x, batch_size, dim, seqlen);
110
+ CHECK_SHAPE(weight, dim, width);
111
+
112
+ TORCH_CHECK(x.stride(2) == 1 || x.stride(1) == 1);
113
+ const bool is_channel_last = x.stride(1) == 1 && x.stride(2) > 1;
114
+
115
+ if (is_channel_last) {
116
+ TORCH_CHECK(dim % 8 == 0, "causal_conv1d only supports channel dimension divisible by 8 for now");
117
+ TORCH_CHECK(x.stride(2) % 8 == 0 and x.stride(0) % 8 == 0, "causal_conv1d with channel last layout requires strides (x.stride(0) and x.stride(2)) to be multiples of 8");
118
+ }
119
+ TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4");
120
+
121
+ if (bias_.has_value()) {
122
+ auto bias = bias_.value();
123
+ TORCH_CHECK(bias.scalar_type() == weight_type);
124
+ TORCH_CHECK(bias.is_cuda());
125
+ TORCH_CHECK(bias.stride(-1) == 1);
126
+ CHECK_SHAPE(bias, dim);
127
+ }
128
+
129
+ if (seq_idx_.has_value()) {
130
+ TORCH_CHECK(is_channel_last, "seq_idx is only supported for channel last layout");
131
+ auto seq_idx = seq_idx_.value();
132
+ TORCH_CHECK(seq_idx.scalar_type() == torch::kInt32);
133
+ TORCH_CHECK(seq_idx.is_cuda());
134
+ TORCH_CHECK(seq_idx.is_contiguous());
135
+ CHECK_SHAPE(seq_idx, batch_size, seqlen);
136
+ }
137
+
138
+ at::Tensor out = torch::empty_like(x);
139
+
140
+ ConvParamsBase params;
141
+ set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
142
+ bias_.has_value() ? bias_.value().data_ptr() : nullptr,
143
+ silu_activation);
144
+
145
+ if (seq_idx_.has_value()) {
146
+ params.seq_idx_ptr = seq_idx_.value().data_ptr();
147
+ } else {
148
+ params.seq_idx_ptr = nullptr;
149
+ }
150
+
151
+ if (initial_states_.has_value()) {
152
+ TORCH_CHECK(is_channel_last, "initial_states is only supported for channel last layout");
153
+ auto initial_states = initial_states_.value();
154
+ TORCH_CHECK(initial_states.scalar_type() == input_type);
155
+ TORCH_CHECK(initial_states.is_cuda());
156
+ CHECK_SHAPE(initial_states, batch_size, dim, width - 1);
157
+ TORCH_CHECK(initial_states.stride(1) == 1);
158
+ params.initial_states_ptr = initial_states.data_ptr();
159
+ params.initial_states_batch_stride = initial_states.stride(0);
160
+ params.initial_states_c_stride = initial_states.stride(1);
161
+ params.initial_states_l_stride = initial_states.stride(2);
162
+ } else {
163
+ params.initial_states_ptr = nullptr;
164
+ }
165
+
166
+ if (final_states_out_.has_value()) {
167
+ TORCH_CHECK(is_channel_last, "final_states is only supported for channel last layout");
168
+ auto final_states = final_states_out_.value();
169
+ TORCH_CHECK(final_states.scalar_type() == input_type);
170
+ TORCH_CHECK(final_states.is_cuda());
171
+ CHECK_SHAPE(final_states, batch_size, dim, width - 1);
172
+ TORCH_CHECK(final_states.stride(1) == 1);
173
+ params.final_states_ptr = final_states.data_ptr();
174
+ params.final_states_batch_stride = final_states.stride(0);
175
+ params.final_states_c_stride = final_states.stride(1);
176
+ params.final_states_l_stride = final_states.stride(2);
177
+ } else {
178
+ params.final_states_ptr = nullptr;
179
+ }
180
+
181
+ // Otherwise the kernel will be launched from cuda:0 device
182
+ // Cast to char to avoid compiler warning about narrowing
183
+ at::cuda::CUDAGuard device_guard{(char)x.get_device()};
184
+ auto stream = at::cuda::getCurrentCUDAStream().stream();
185
+ DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_fwd", [&] {
186
+ if (!is_channel_last) {
187
+ causal_conv1d_fwd_cuda<input_t, weight_t>(params, stream);
188
+ } else {
189
+ causal_conv1d_channellast_fwd_cuda<input_t, weight_t>(params, stream);
190
+ }
191
+ });
192
+ return out;
193
+}
194
+
195
+
196
+at::Tensor
197
+causal_conv1d_update(const at::Tensor &x,
198
+ const at::Tensor &conv_state,
199
+ const at::Tensor &weight,
200
+ const c10::optional<at::Tensor> &bias_,
201
+ bool silu_activation) {
202
+ auto input_type = x.scalar_type();
203
+ auto weight_type = weight.scalar_type();
204
+ TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
205
+ TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16);
206
+ TORCH_CHECK(weight_type == input_type, "weight type must equal to input type, other variations are disabled due to binary size limitations");
207
+ TORCH_CHECK(conv_state.scalar_type() == input_type);
208
+
209
+ TORCH_CHECK(x.is_cuda());
210
+ TORCH_CHECK(conv_state.is_cuda());
211
+ TORCH_CHECK(weight.is_cuda());
212
+
213
+ const auto sizes = x.sizes();
214
+ const int batch_size = sizes[0];
215
+ const int dim = sizes[1];
216
+ const int width = weight.size(-1);
217
+
218
+ CHECK_SHAPE(x, batch_size, dim);
219
+ CHECK_SHAPE(conv_state, batch_size, dim, width);
220
+ CHECK_SHAPE(weight, dim, width);
221
+
222
+ TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4");
223
+
224
+ if (bias_.has_value()) {
225
+ auto bias = bias_.value();
226
+ TORCH_CHECK(bias.scalar_type() == weight_type);
227
+ TORCH_CHECK(bias.is_cuda());
228
+ TORCH_CHECK(bias.stride(-1) == 1);
229
+ CHECK_SHAPE(bias, dim);
230
+ }
231
+
232
+ at::Tensor out = torch::empty_like(x);
233
+
234
+ ConvParamsBase params;
235
+ set_conv_params_fwd(params, batch_size, dim, /*seqlen=*/1, width, x, weight, out,
236
+ bias_.has_value() ? bias_.value().data_ptr() : nullptr,
237
+ silu_activation);
238
+ params.conv_state_ptr = conv_state.data_ptr();
239
+ // All stride are in elements, not bytes.
240
+ params.conv_state_batch_stride = conv_state.stride(0);
241
+ params.conv_state_c_stride = conv_state.stride(1);
242
+ params.conv_state_l_stride = conv_state.stride(2);
243
+
244
+ // Otherwise the kernel will be launched from cuda:0 device
245
+ // Cast to char to avoid compiler warning about narrowing
246
+ at::cuda::CUDAGuard device_guard{(char)x.get_device()};
247
+ auto stream = at::cuda::getCurrentCUDAStream().stream();
248
+ DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_update", [&] {
249
+ causal_conv1d_update_cuda<input_t, weight_t>(params, stream);
250
+ });
251
+ return out;
252
+}
253
+
254
+template<int kNThreads_, int kWidth_, bool kIsVecLoad_, typename input_t_, typename weight_t_>
255
+struct Causal_conv1d_fwd_kernel_traits {
256
+ using input_t = input_t_;
257
+ using weight_t = weight_t_;
258
+ static constexpr int kNThreads = kNThreads_;
259
+ static constexpr int kWidth = kWidth_;
260
+ static constexpr int kNBytes = sizeof(input_t);
261
+ static_assert(kNBytes == 2 || kNBytes == 4);
262
+ static constexpr int kNElts = kNBytes == 4 ? 4 : 8;
263
+ static_assert(kWidth <= kNElts);
264
+ static constexpr bool kIsVecLoad = kIsVecLoad_;
265
+ using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
266
+ using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNElts, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
267
+ using BlockLoadVecT = cub::BlockLoad<vec_t, kNThreads, 1, cub::BLOCK_LOAD_DIRECT>;
268
+ using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNElts, cub::BLOCK_STORE_WARP_TRANSPOSE>;
269
+ using BlockStoreVecT = cub::BlockStore<vec_t, kNThreads, 1, cub::BLOCK_STORE_DIRECT>;
270
+ static constexpr int kSmemIOSize = kIsVecLoad
271
+ ? 0
272
+ : custom_max({sizeof(typename BlockLoadT::TempStorage), sizeof(typename BlockStoreT::TempStorage)});
273
+ static constexpr int kSmemExchangeSize = kNThreads * kNBytes * kNElts;
274
+ static constexpr int kSmemSize = kSmemIOSize + kSmemExchangeSize;
275
+};
276
+
277
+template<typename Ktraits>
278
+__global__ __launch_bounds__(Ktraits::kNThreads)
279
+void causal_conv1d_fwd_kernel(ConvParamsBase params) {
280
+ constexpr int kWidth = Ktraits::kWidth;
281
+ constexpr int kNThreads = Ktraits::kNThreads;
282
+ constexpr int kNElts = Ktraits::kNElts;
283
+ static constexpr bool kIsVecLoad = Ktraits::kIsVecLoad;
284
+ using input_t = typename Ktraits::input_t;
285
+ using vec_t = typename Ktraits::vec_t;
286
+ using weight_t = typename Ktraits::weight_t;
287
+
288
+ // Shared memory.
289
+ extern __shared__ char smem_[];
290
+ auto& smem_load = reinterpret_cast<typename Ktraits::BlockLoadT::TempStorage&>(smem_);
291
+ auto& smem_load_vec = reinterpret_cast<typename Ktraits::BlockLoadVecT::TempStorage&>(smem_);
292
+ auto& smem_store = reinterpret_cast<typename Ktraits::BlockStoreT::TempStorage&>(smem_);
293
+ auto& smem_store_vec = reinterpret_cast<typename Ktraits::BlockStoreVecT::TempStorage&>(smem_);
294
+ vec_t *smem_exchange = reinterpret_cast<vec_t *>(smem_ + Ktraits::kSmemIOSize);
295
+
296
+ const int tidx = threadIdx.x;
297
+ const int batch_id = blockIdx.x;
298
+ const int channel_id = blockIdx.y;
299
+ input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
300
+ + channel_id * params.x_c_stride;
301
+ weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr) + channel_id * params.weight_c_stride;
302
+ input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
303
+ + channel_id * params.out_c_stride;
304
+ float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[channel_id]);
305
+
306
+ // Thread 0 will load the last elements of the previous chunk, so we initialize those to 0.
307
+ if (tidx == 0) {
308
+ input_t zeros[kNElts] = {0};
309
+ smem_exchange[kNThreads - 1] = reinterpret_cast<vec_t *>(zeros)[0];
310
+ }
311
+
312
+ float weight_vals[kWidth];
313
+ #pragma unroll
314
+ for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); }
315
+
316
+ constexpr int kChunkSize = kNThreads * kNElts;
317
+ const int n_chunks = (params.seqlen + kChunkSize - 1) / kChunkSize;
318
+ for (int chunk = 0; chunk < n_chunks; ++chunk) {
319
+ input_t x_vals_load[2 * kNElts] = {0};
320
+ if constexpr(kIsVecLoad) {
321
+ typename Ktraits::BlockLoadVecT(smem_load_vec).Load(reinterpret_cast<vec_t*>(x), *reinterpret_cast<vec_t (*)[1]>(&x_vals_load[kNElts]), (params.seqlen - chunk * kChunkSize) / kNElts);
322
+ } else {
323
+ __syncthreads();
324
+ typename Ktraits::BlockLoadT(smem_load).Load(x, *reinterpret_cast<input_t (*)[kNElts]>(&x_vals_load[kNElts]), params.seqlen - chunk * kChunkSize);
325
+ }
326
+ x += kChunkSize;
327
+ __syncthreads();
328
+ // Thread kNThreads - 1 don't write yet, so that thread 0 can read
329
+ // the last elements of the previous chunk.
330
+ if (tidx < kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast<vec_t *>(x_vals_load)[1]; }
331
+ __syncthreads();
332
+ reinterpret_cast<vec_t *>(x_vals_load)[0] = smem_exchange[tidx > 0 ? tidx - 1 : kNThreads - 1];
333
+ __syncthreads();
334
+ // Now thread kNThreads - 1 can write the last elements of the current chunk.
335
+ if (tidx == kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast<vec_t *>(x_vals_load)[1]; }
336
+
337
+ float x_vals[2 * kNElts];
338
+ #pragma unroll
339
+ for (int i = 0; i < 2 * kNElts; ++i) { x_vals[i] = float(x_vals_load[i]); }
340
+
341
+ float out_vals[kNElts];
342
+ #pragma unroll
343
+ for (int i = 0; i < kNElts; ++i) {
344
+ out_vals[i] = bias_val;
345
+ #pragma unroll
346
+ for (int w = 0; w < kWidth; ++w) {
347
+ out_vals[i] += weight_vals[w] * x_vals[kNElts + i - (kWidth - w - 1)];
348
+ }
349
+ }
350
+
351
+ if (params.silu_activation) {
352
+ #pragma unroll
353
+ for (int i = 0; i < kNElts; ++i) {
354
+ out_vals[i] = out_vals[i] / (1 + expf(-out_vals[i]));
355
+ }
356
+ }
357
+
358
+ input_t out_vals_store[kNElts];
359
+ #pragma unroll
360
+ for (int i = 0; i < kNElts; ++i) { out_vals_store[i] = out_vals[i]; }
361
+ if constexpr(kIsVecLoad) {
362
+ typename Ktraits::BlockStoreVecT(smem_store_vec).Store(reinterpret_cast<vec_t*>(out), reinterpret_cast<vec_t (&)[1]>(out_vals_store), (params.seqlen - chunk * kChunkSize) / kNElts);
363
+ } else {
364
+ typename Ktraits::BlockStoreT(smem_store).Store(out, out_vals_store, params.seqlen - chunk * kChunkSize);
365
+ }
366
+ out += kChunkSize;
367
+ }
368
+}
369
+
370
+
371
+template<int kNThreads, int kWidth, typename input_t, typename weight_t>
372
+void causal_conv1d_fwd_launch(ConvParamsBase ¶ms, cudaStream_t stream) {
373
+ static constexpr int kNElts = sizeof(input_t) == 4 ? 4 : 8;
374
+ BOOL_SWITCH(params.seqlen % kNElts == 0, kIsVecLoad, [&] {
375
+ using Ktraits = Causal_conv1d_fwd_kernel_traits<kNThreads, kWidth, kIsVecLoad, input_t, weight_t>;
376
+ constexpr int kSmemSize = Ktraits::kSmemSize;
377
+ dim3 grid(params.batch, params.dim);
378
+
379
+ auto kernel = &causal_conv1d_fwd_kernel<Ktraits>;
380
+
381
+ if (kSmemSize >= 48 * 1024) {
382
+ #ifndef USE_ROCM
383
+ C10_CUDA_CHECK(cudaFuncSetAttribute(
384
+ kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
385
+ #else
386
+ // There is a slight signature discrepancy in HIP and CUDA "FuncSetAttribute" function.
387
+ C10_CUDA_CHECK(cudaFuncSetAttribute(
388
+ (void *) kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
389
+ std::cerr << "Warning (causal_conv1d fwd launch): attempting to set maxDynamicSharedMemorySize on an AMD GPU which is currently a non-op (in ROCm versions <= 6.1). This might lead to undefined behavior. \n" << std::endl;
390
+ #endif
391
+ }
392
+ kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
393
+
394
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
395
+ });
396
+}
397
+
398
+template<typename input_t, typename weight_t>
399
+void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream) {
400
+ if (params.width == 2) {
401
+ causal_conv1d_fwd_launch<128, 2, input_t, weight_t>(params, stream);
402
+ } else if (params.width == 3) {
403
+ causal_conv1d_fwd_launch<128, 3, input_t, weight_t>(params, stream);
404
+ } else if (params.width == 4) {
405
+ causal_conv1d_fwd_launch<128, 4, input_t, weight_t>(params, stream);
406
+ }
407
+}
408
+
409
+template<int kNThreads_, int kWidth_, int kChunkSizeL_, bool kIsVecLoad_, typename input_t_, typename weight_t_>
410
+struct Causal_conv1d_channellast_fwd_kernel_traits {
411
+ // The cache line is 128 bytes, and we try to read 16 bytes per thread.
412
+ // So we have 8 threads per "row", so 32 or 64 elements in the channel dimension.
413
+ // That leaves 4 columns per warp, and so 16 columns per block (assuming each block has 128
414
+ // threads). Each each load is 16 x 32|64 elements in the L x C dimensions.
415
+ using input_t = input_t_;
416
+ using weight_t = weight_t_;
417
+ static constexpr int kNThreads = kNThreads_;
418
+ static_assert(kNThreads % 32 == 0);
419
+ static constexpr int kNWarps = kNThreads / 32;
420
+ static constexpr int kWidth = kWidth_;
421
+ static constexpr int kChunkSizeL = kChunkSizeL_;
422
+ static constexpr int kNBytes = sizeof(input_t);
423
+ static_assert(kNBytes == 2 || kNBytes == 4);
424
+ static constexpr int kNElts = kNBytes == 4 ? 4 : 8;
425
+ static constexpr int kNEltsPerRow = 128 / kNBytes;
426
+ static constexpr int kNThreadsPerRow = kNEltsPerRow / kNElts; // Always 8 for now
427
+ static_assert(kNThreadsPerRow * kNBytes * kNElts == 128);
428
+ static constexpr int kNColsPerWarp = 32 / kNThreadsPerRow; // Always 4 for now
429
+ static_assert(kNColsPerWarp * kNThreadsPerRow == 32);
430
+ static constexpr int kNColsPerLoad = kNColsPerWarp * kNWarps;
431
+ static constexpr int kNLoads = kChunkSizeL / kNColsPerLoad;
432
+ static_assert(kNLoads * kNColsPerLoad == kChunkSizeL);
433
+ static constexpr bool kIsVecLoad = kIsVecLoad_;
434
+ using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
435
+ // using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
436
+ // using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNItems, cub::BLOCK_STORE_WARP_TRANSPOSE>;
437
+ // static constexpr int kSmemSize = std::max({sizeof(typename BlockLoadT::TempStorage),
438
+ // sizeof(typename BlockStoreT::TempStorage)});
439
+ // static constexpr int kSmemSize = kChunkSizeL * kNEltsPerRow * kNBytes;
440
+};
441
+
442
+template<typename Ktraits, bool kHasSeqIdx>
443
+__global__ __launch_bounds__(Ktraits::kNThreads)
444
+void causal_conv1d_channellast_fwd_kernel(ConvParamsBase params) {
445
+ constexpr int kWidth = Ktraits::kWidth;
446
+ constexpr int kNThreads = Ktraits::kNThreads;
447
+ constexpr int kNElts = Ktraits::kNElts;
448
+ constexpr int kNThreadsPerC = Ktraits::kNThreadsPerRow;
449
+ constexpr int kLPerLoad = Ktraits::kNColsPerLoad;
450
+ constexpr int kChunkSizeL = Ktraits::kChunkSizeL;
451
+ constexpr int kChunkSizeC = Ktraits::kNEltsPerRow;
452
+ using input_t = typename Ktraits::input_t;
453
+ using vec_t = typename Ktraits::vec_t;
454
+ using weight_t = typename Ktraits::weight_t;
455
+
456
+ // Shared memory.
457
+ __shared__ input_t x_smem[kWidth - 1 + kChunkSizeL][kChunkSizeC + kNElts];
458
+
459
+ const int batch_id = blockIdx.x;
460
+ const int chunk_l_id = blockIdx.y;
461
+ const int chunk_c_id = blockIdx.z;
462
+ const int tid = threadIdx.x;
463
+ const int l_idx = tid / kNThreadsPerC;
464
+ const int c_idx = tid % kNThreadsPerC;
465
+ input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
466
+ + (chunk_l_id * kChunkSizeL + l_idx) * params.x_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
467
+ weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr)
468
+ + chunk_c_id * kChunkSizeC * params.weight_c_stride;
469
+ input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
470
+ + (chunk_l_id * kChunkSizeL + l_idx) * params.out_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
471
+ int *seq_idx = !kHasSeqIdx ? nullptr : reinterpret_cast<int *>(params.seq_idx_ptr)
472
+ + batch_id * params.seqlen + chunk_l_id * kChunkSizeL;
473
+ input_t *initial_states = params.initial_states_ptr == nullptr || chunk_l_id > 0 ? nullptr
474
+ : reinterpret_cast<input_t *>(params.initial_states_ptr) + batch_id * params.initial_states_batch_stride + l_idx * params.initial_states_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
475
+ // The last L-chunk will also have enough info to write to final states, since it also contain a few x values
476
+ // from the previous L-chunk.
477
+ input_t *final_states = params.final_states_ptr == nullptr || chunk_l_id < gridDim.y - 1 ? nullptr
478
+ : reinterpret_cast<input_t *>(params.final_states_ptr) + batch_id * params.final_states_batch_stride + l_idx * params.final_states_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
479
+
480
+ #pragma unroll
481
+ for (int l = 0; l < Ktraits::kNLoads; ++l) {
482
+ input_t x_vals_load[kNElts] = {0};
483
+ if (chunk_l_id * kChunkSizeL + l * kLPerLoad + l_idx < params.seqlen
484
+ && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
485
+ reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(x + l * kLPerLoad * params.x_l_stride);
486
+ }
487
+ reinterpret_cast<vec_t *>(x_smem[kWidth - 1 + l * kLPerLoad + l_idx])[c_idx] = reinterpret_cast<vec_t *>(x_vals_load)[0];
488
+ }
489
+ // Load the elements from the previous chunk that are needed for convolution.
490
+ if (l_idx < kWidth - 1) {
491
+ input_t x_vals_load[kNElts] = {0};
492
+ if (chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) >= 0
493
+ && chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) < params.seqlen
494
+ && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
495
+ reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(x - (kWidth - 1) * params.x_l_stride);
496
+ } else if (initial_states != nullptr
497
+ && chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) < 0
498
+ && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
499
+ reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(initial_states);
500
+ }
501
+ reinterpret_cast<vec_t *>(x_smem[l_idx])[c_idx] = reinterpret_cast<vec_t *>(x_vals_load)[0];
502
+ }
503
+
504
+ __syncthreads();
505
+
506
+ if (final_states != nullptr
507
+ && l_idx < kWidth - 1
508
+ && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
509
+ // x_smem[0] contains element at index chunk_l_id * kChunkSizeL - (kWidth - 1)
510
+ // So last few elements (index params.seqlen - kWidth + 1 + l_idx) are stored in x_smem[params.seqlen - kWidth + 1 + l_idx - (chunk_l_id * kChunkSizeL - kWidth + 1)][c_idx]
511
+ *reinterpret_cast<vec_t *>(final_states) = reinterpret_cast<vec_t *>(x_smem[params.seqlen + l_idx - chunk_l_id * kChunkSizeL])[c_idx];
512
+ }
513
+
514
+ constexpr int kLPerThread = constexpr_min(kChunkSizeL * kChunkSizeC / kNThreads, kChunkSizeL);
515
+ static_assert(kLPerThread * kNThreads == kChunkSizeL * kChunkSizeC);
516
+ constexpr int kNThreadsPerRow = kChunkSizeL / kLPerThread;
517
+ static_assert(kNThreadsPerRow * kLPerThread == kChunkSizeL);
518
+ // kChunkSizeL, kLPerThread, kNThreadsPerRow should be powers of 2 for simplicity
519
+ static_assert((kChunkSizeL & (kChunkSizeL - 1)) == 0);
520
+ static_assert((kLPerThread & (kLPerThread - 1)) == 0);
521
+ static_assert((kNThreadsPerRow & (kNThreadsPerRow - 1)) == 0);
522
+ static_assert(kNThreadsPerRow <= 32);
523
+
524
+ const int row_idx = tid / kNThreadsPerRow;
525
+ const int col_idx = tid % kNThreadsPerRow;
526
+
527
+ float bias_val = params.bias_ptr == nullptr || chunk_c_id * kChunkSizeC + row_idx >= params.dim ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[chunk_c_id * kChunkSizeC + row_idx]);
528
+ float weight_vals[kWidth] = {0};
529
+ if (chunk_c_id * kChunkSizeC + row_idx < params.dim) {
530
+ #pragma unroll
531
+ for (int w = 0; w < kWidth; ++w) {
532
+ weight_vals[w] = weight[row_idx * params.weight_c_stride + w * params.weight_width_stride];
533
+ }
534
+ }
535
+ float x_vals[kWidth - 1 + kLPerThread];
536
+ #pragma unroll
537
+ for (int i = 0; i < kWidth - 1 + kLPerThread; ++i) {
538
+ x_vals[i] = float(x_smem[col_idx * kLPerThread + i][row_idx]);
539
+ }
540
+ int seq_idx_thread[kWidth - 1 + kLPerThread];
541
+ if constexpr (kHasSeqIdx) {
542
+ #pragma unroll
543
+ for (int i = 0; i < kWidth - 1 + kLPerThread; ++i) {
544
+ seq_idx_thread[i] = chunk_l_id * kChunkSizeL + col_idx * kLPerThread + i - (kWidth - 1) >= 0 ? seq_idx[col_idx * kLPerThread + i - (kWidth - 1)] : -1;
545
+ }
546
+ }
547
+
548
+ float out_vals[kLPerThread];
549
+ #pragma unroll
550
+ for (int i = 0; i < kLPerThread; ++i) {
551
+ out_vals[i] = bias_val;
552
+ const int seq_idx_cur = !kHasSeqIdx ? 0 : seq_idx_thread[i + kWidth - 1];
553
+ #pragma unroll
554
+ for (int w = 0; w < kWidth; ++w) {
555
+ if constexpr (!kHasSeqIdx) {
556
+ out_vals[i] += weight_vals[w] * x_vals[i + w];
557
+ } else {
558
+ out_vals[i] += seq_idx_thread[i + w] == seq_idx_cur ? weight_vals[w] * x_vals[i + w] : 0.f;
559
+ }
560
+ }
561
+ if (params.silu_activation) {out_vals[i] = out_vals[i] / (1 + expf(-out_vals[i])); }
562
+ }
563
+
564
+ __syncthreads();
565
+ #pragma unroll
566
+ for (int i = 0; i < kLPerThread; ++i) { x_smem[col_idx * kLPerThread + i][row_idx] = out_vals[i]; }
567
+ __syncthreads();
568
+
569
+ #pragma unroll
570
+ for (int l = 0; l < Ktraits::kNLoads; ++l) {
571
+ input_t out_vals_store[kNElts];
572
+ reinterpret_cast<vec_t *>(out_vals_store)[0] = reinterpret_cast<vec_t *>(x_smem[l * kLPerLoad + l_idx])[c_idx];
573
+ if (chunk_l_id * kChunkSizeL + l * kLPerLoad + l_idx < params.seqlen
574
+ && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
575
+ *reinterpret_cast<vec_t *>(out + l * kLPerLoad * params.out_l_stride) = reinterpret_cast<vec_t *>(out_vals_store)[0];
576
+ }
577
+ }
578
+
579
+}
580
+
581
+template<int kNThreads, int kWidth, typename input_t, typename weight_t>
582
+void causal_conv1d_channellast_fwd_launch(ConvParamsBase ¶ms, cudaStream_t stream) {
583
+ BOOL_SWITCH(params.seq_idx_ptr != nullptr, kHasSeqIdx, [&] {
584
+ using Ktraits = Causal_conv1d_channellast_fwd_kernel_traits<kNThreads, kWidth, 64, true, input_t, weight_t>;
585
+ // constexpr int kSmemSize = Ktraits::kSmemSize;
586
+ constexpr int kChunkSizeL = Ktraits::kChunkSizeL;
587
+ constexpr int kChunkSizeC = Ktraits::kNEltsPerRow;
588
+ const int n_chunks_L = (params.seqlen + kChunkSizeL - 1) / kChunkSizeL;
589
+ const int n_chunks_C = (params.dim + kChunkSizeC - 1) / kChunkSizeC;
590
+ dim3 grid(params.batch, n_chunks_L, n_chunks_C);
591
+ dim3 block(Ktraits::kNThreads);
592
+ auto kernel = &causal_conv1d_channellast_fwd_kernel<Ktraits, kHasSeqIdx>;
593
+ // if (kSmemSize >= 48 * 1024) {
594
+ // C10_CUDA_CHECK(cudaFuncSetAttribute(
595
+ // kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
596
+ // }
597
+ // kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
598
+ kernel<<<grid, Ktraits::kNThreads, 0, stream>>>(params);
599
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
600
+ });
601
+}
602
+
603
+template<typename input_t, typename weight_t>
604
+void causal_conv1d_channellast_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream) {
605
+ if (params.width == 2) {
606
+ causal_conv1d_channellast_fwd_launch<128, 2, input_t, weight_t>(params, stream);
607
+ } else if (params.width == 3) {
608
+ causal_conv1d_channellast_fwd_launch<128, 3, input_t, weight_t>(params, stream);
609
+ } else if (params.width == 4) {
610
+ causal_conv1d_channellast_fwd_launch<128, 4, input_t, weight_t>(params, stream);
611
+ }
612
+}
613
+
614
+template void causal_conv1d_fwd_cuda<float, float>(ConvParamsBase ¶ms, cudaStream_t stream);
615
+template void causal_conv1d_fwd_cuda<at::Half, at::Half>(ConvParamsBase ¶ms, cudaStream_t stream);
616
+template void causal_conv1d_fwd_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase ¶ms, cudaStream_t stream);
617
+
618
+template void causal_conv1d_channellast_fwd_cuda<float, float>(ConvParamsBase ¶ms, cudaStream_t stream);
619
+template void causal_conv1d_channellast_fwd_cuda<at::Half, at::Half>(ConvParamsBase ¶ms, cudaStream_t stream);
620
+template void causal_conv1d_channellast_fwd_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase ¶ms, cudaStream_t stream);
621
+///////
622
+
623
+
624
+
625
+
626
+template<int kNThreads_, int kWidth_, typename input_t_, typename weight_t_>
627
+struct Causal_conv1d_update_kernel_traits {
628
+ using input_t = input_t_;
629
+ using weight_t = weight_t_;
630
+ static constexpr int kNThreads = kNThreads_;
631
+ static constexpr int kWidth = kWidth_;
632
+ static constexpr int kNBytes = sizeof(input_t);
633
+ static_assert(kNBytes == 2 || kNBytes == 4);
634
+};
635
+
636
+template<typename Ktraits>
637
+__global__ __launch_bounds__(Ktraits::kNThreads)
638
+void causal_conv1d_update_kernel(ConvParamsBase params) {
639
+ constexpr int kWidth = Ktraits::kWidth;
640
+ constexpr int kNThreads = Ktraits::kNThreads;
641
+ using input_t = typename Ktraits::input_t;
642
+ using weight_t = typename Ktraits::weight_t;
643
+
644
+ const int tidx = threadIdx.x;
645
+ const int batch_id = blockIdx.x;
646
+ const int channel_id = blockIdx.y * kNThreads + tidx;
647
+ input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
648
+ + channel_id * params.x_c_stride;
649
+ input_t *conv_state = reinterpret_cast<input_t *>(params.conv_state_ptr) + batch_id * params.conv_state_batch_stride
650
+ + channel_id * params.conv_state_c_stride;
651
+ weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr) + channel_id * params.weight_c_stride;
652
+ input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
653
+ + channel_id * params.out_c_stride;
654
+ float bias_val = params.bias_ptr == nullptr || channel_id >= params.dim ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[channel_id]);
655
+
656
+ float weight_vals[kWidth] = {0};
657
+ if (channel_id < params.dim) {
658
+ #pragma unroll
659
+ for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); }
660
+ }
661
+
662
+ float x_vals[kWidth] = {0};
663
+ if (channel_id < params.dim) {
664
+ #pragma unroll
665
+ for (int i = 0; i < kWidth - 1; ++i) { x_vals[i] = float(conv_state[(i + 1) * params.conv_state_l_stride]); }
666
+ x_vals[kWidth - 1] = float(x[0]);
667
+ #pragma unroll
668
+ for (int i = 0; i < kWidth; ++i) { conv_state[i * params.conv_state_l_stride] = input_t(x_vals[i]); }
669
+ }
670
+
671
+ float out_val = bias_val;
672
+ #pragma unroll
673
+ for (int i = 0; i < kWidth; ++i) { out_val += weight_vals[i] * x_vals[i]; }
674
+ if (params.silu_activation) { out_val = out_val / (1 + expf(-out_val)); }
675
+ if (channel_id < params.dim) { out[0] = input_t(out_val); }
676
+}
677
+
678
+template<int kNThreads, int kWidth, typename input_t, typename weight_t>
679
+void causal_conv1d_update_launch(ConvParamsBase ¶ms, cudaStream_t stream) {
680
+ using Ktraits = Causal_conv1d_update_kernel_traits<kNThreads, kWidth, input_t, weight_t>;
681
+ dim3 grid(params.batch, (params.dim + kNThreads - 1) / kNThreads);
682
+ auto kernel = &causal_conv1d_update_kernel<Ktraits>;
683
+ kernel<<<grid, Ktraits::kNThreads, 0, stream>>>(params);
684
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
685
+}
686
+
687
+template<typename input_t, typename weight_t>
688
+void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream) {
689
+ if (params.width == 2) {
690
+ causal_conv1d_update_launch<64, 2, input_t, weight_t>(params, stream);
691
+ } else if (params.width == 3) {
692
+ causal_conv1d_update_launch<64, 3, input_t, weight_t>(params, stream);
693
+ } else if (params.width == 4) {
694
+ causal_conv1d_update_launch<64, 4, input_t, weight_t>(params, stream);
695
+ }
696
+}
697
+
698
+template void causal_conv1d_update_cuda<float, float>(ConvParamsBase ¶ms, cudaStream_t stream);
699
+template void causal_conv1d_update_cuda<at::Half, at::Half>(ConvParamsBase ¶ms, cudaStream_t stream);
700
+template void causal_conv1d_update_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase ¶ms, cudaStream_t stream);
csrc/mamba/causal_conv1d/causal_conv1d.h
new
+144
@@ -0,0 +1,144 @@
1
+/******************************************************************************
2
+ * Copyright (c) 2024, Tri Dao.
3
+ ******************************************************************************/
4
+// clang-format off
5
+// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d.h
6
+#pragma once
7
+
8
+#include <cuda_bf16.h>
9
+#include <cuda_fp16.h>
10
+////////////////////////////////////////////////////////////////////////////////////////////////////
11
+
12
+struct ConvParamsBase {
13
+ using index_t = uint32_t;
14
+
15
+ int batch, dim, seqlen, width;
16
+ bool silu_activation;
17
+
18
+ index_t x_batch_stride;
19
+ index_t x_c_stride;
20
+ index_t x_l_stride;
21
+ index_t weight_c_stride;
22
+ index_t weight_width_stride;
23
+ index_t out_batch_stride;
24
+ index_t out_c_stride;
25
+ index_t out_l_stride;
26
+
27
+ index_t conv_state_batch_stride;
28
+ index_t conv_state_c_stride;
29
+ index_t conv_state_l_stride;
30
+
31
+ // Common data pointers.
32
+ void *__restrict__ x_ptr;
33
+ void *__restrict__ weight_ptr;
34
+ void *__restrict__ bias_ptr;
35
+ void *__restrict__ out_ptr;
36
+
37
+ void *__restrict__ conv_state_ptr;
38
+
39
+ void *__restrict__ seq_idx_ptr;
40
+
41
+ // No __restrict__ since initial_states could be the same as final_states.
42
+ void * initial_states_ptr;
43
+ index_t initial_states_batch_stride;
44
+ index_t initial_states_l_stride;
45
+ index_t initial_states_c_stride;
46
+
47
+ void * final_states_ptr;
48
+ index_t final_states_batch_stride;
49
+ index_t final_states_l_stride;
50
+ index_t final_states_c_stride;
51
+};
52
+
53
+
54
+#ifndef USE_ROCM
55
+ #include <cuda_bf16.h>
56
+
57
+ template<typename T>
58
+ __device__ inline T shuffle_xor(T val, int offset) {
59
+ return __shfl_xor_sync(uint32_t(-1), val, offset);
60
+ }
61
+
62
+ constexpr size_t custom_max(std::initializer_list<size_t> ilist)
63
+ {
64
+ return std::max(ilist);
65
+ }
66
+
67
+ template<typename T>
68
+ constexpr T constexpr_min(T a, T b) {
69
+ return std::min(a, b);
70
+ }
71
+
72
+#else
73
+ #include <hip/hip_bf16.h>
74
+
75
+ template<typename T>
76
+ __device__ inline T shuffle_xor(T val, int offset) {
77
+ return __shfl_xor(val, offset);
78
+ }
79
+ constexpr size_t custom_max(std::initializer_list<size_t> ilist)
80
+ {
81
+ return *std::max_element(ilist.begin(), ilist.end());
82
+ }
83
+
84
+ template<typename T>
85
+ constexpr T constexpr_min(T a, T b) {
86
+ return a < b ? a : b;
87
+ }
88
+#endif
89
+
90
+////////////////////////////////////////////////////////////////////////////////////////////////////
91
+
92
+template<int BYTES> struct BytesToType {};
93
+
94
+template<> struct BytesToType<16> {
95
+ using Type = uint4;
96
+ static_assert(sizeof(Type) == 16);
97
+};
98
+
99
+template<> struct BytesToType<8> {
100
+ using Type = uint64_t;
101
+ static_assert(sizeof(Type) == 8);
102
+};
103
+
104
+template<> struct BytesToType<4> {
105
+ using Type = uint32_t;
106
+ static_assert(sizeof(Type) == 4);
107
+};
108
+
109
+template<> struct BytesToType<2> {
110
+ using Type = uint16_t;
111
+ static_assert(sizeof(Type) == 2);
112
+};
113
+
114
+template<> struct BytesToType<1> {
115
+ using Type = uint8_t;
116
+ static_assert(sizeof(Type) == 1);
117
+};
118
+
119
+////////////////////////////////////////////////////////////////////////////////////////////////////
120
+
121
+template<typename T>
122
+struct SumOp {
123
+__device__ inline T operator()(T const & x, T const & y) { return x + y; }
124
+};
125
+
126
+template<int THREADS>
127
+struct Allreduce {
128
+ static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4);
129
+ template<typename T, typename Operator>
130
+ static __device__ inline T run(T x, Operator &op) {
131
+ constexpr int OFFSET = THREADS / 2;
132
+ x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET));
133
+ return Allreduce<OFFSET>::run(x, op);
134
+ }
135
+};
136
+
137
+template<>
138
+struct Allreduce<2> {
139
+template<typename T, typename Operator>
140
+static __device__ inline T run(T x, Operator &op) {
141
+ x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1));
142
+ return x;
143
+}
144
+};
csrc/mamba/causal_conv1d/static_switch.h
new
+28
@@ -0,0 +1,28 @@
1
+// Inspired by
2
+// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h
3
+// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h
4
+// clang-format off
5
+// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/static_switch.h
6
+
7
+#pragma once
8
+
9
+/// @param COND - a boolean expression to switch by
10
+/// @param CONST_NAME - a name given for the constexpr bool variable.
11
+/// @param ... - code to execute for true and false
12
+///
13
+/// Usage:
14
+/// ```
15
+/// BOOL_SWITCH(flag, BoolConst, [&] {
16
+/// some_function<BoolConst>(...);
17
+/// });
18
+/// ```
19
+#define BOOL_SWITCH(COND, CONST_NAME, ...) \
20
+ [&] { \
21
+ if (COND) { \
22
+ static constexpr bool CONST_NAME = true; \
23
+ return __VA_ARGS__(); \
24
+ } else { \
25
+ static constexpr bool CONST_NAME = false; \
26
+ return __VA_ARGS__(); \
27
+ } \
28
+ }()
csrc/mamba/mamba_ssm/selective_scan.h
new
+276
@@ -0,0 +1,276 @@
1
+/******************************************************************************
2
+ * Copyright (c) 2023, Tri Dao.
3
+ ******************************************************************************/
4
+// clang-format off
5
+// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/selective_scan.h
6
+
7
+#pragma once
8
+
9
+#ifndef USE_ROCM
10
+ #include <cuda_bf16.h>
11
+#else
12
+ #include <hip/hip_bf16.h>
13
+#endif
14
+#include <cuda_fp16.h>
15
+////////////////////////////////////////////////////////////////////////////////////////////////////
16
+
17
+struct SSMParamsBase {
18
+ using index_t = uint32_t;
19
+
20
+ int batch, dim, seqlen, dstate, n_groups, n_chunks;
21
+ int dim_ngroups_ratio;
22
+ bool is_variable_B;
23
+ bool is_variable_C;
24
+
25
+ bool delta_softplus;
26
+
27
+ index_t A_d_stride;
28
+ index_t A_dstate_stride;
29
+ index_t B_batch_stride;
30
+ index_t B_d_stride;
31
+ index_t B_dstate_stride;
32
+ index_t B_group_stride;
33
+ index_t C_batch_stride;
34
+ index_t C_d_stride;
35
+ index_t C_dstate_stride;
36
+ index_t C_group_stride;
37
+ index_t u_batch_stride;
38
+ index_t u_d_stride;
39
+ index_t delta_batch_stride;
40
+ index_t delta_d_stride;
41
+ index_t z_batch_stride;
42
+ index_t z_d_stride;
43
+ index_t out_batch_stride;
44
+ index_t out_d_stride;
45
+ index_t out_z_batch_stride;
46
+ index_t out_z_d_stride;
47
+
48
+ // Common data pointers.
49
+ void *__restrict__ A_ptr;
50
+ void *__restrict__ B_ptr;
51
+ void *__restrict__ C_ptr;
52
+ void *__restrict__ D_ptr;
53
+ void *__restrict__ u_ptr;
54
+ void *__restrict__ delta_ptr;
55
+ void *__restrict__ delta_bias_ptr;
56
+ void *__restrict__ out_ptr;
57
+ void *__restrict__ x_ptr;
58
+ void *__restrict__ z_ptr;
59
+ void *__restrict__ out_z_ptr;
60
+ void *__restrict__ index_ptr;
61
+};
62
+
63
+
64
+
65
+
66
+#ifndef USE_ROCM
67
+
68
+ constexpr size_t custom_max(std::initializer_list<size_t> ilist)
69
+ {
70
+ return std::max(ilist);
71
+ }
72
+
73
+ template<typename T>
74
+ constexpr T constexpr_min(T a, T b) {
75
+ return std::min(a, b);
76
+ }
77
+
78
+#else
79
+ constexpr size_t custom_max(std::initializer_list<size_t> ilist)
80
+ {
81
+ return *std::max_element(ilist.begin(), ilist.end());
82
+ }
83
+
84
+ template<typename T>
85
+ constexpr T constexpr_min(T a, T b) {
86
+ return a < b ? a : b;
87
+ }
88
+#endif
89
+
90
+
91
+#define MAX_DSTATE 256
92
+
93
+
94
+inline __device__ float2 operator+(const float2 & a, const float2 & b){
95
+ return {a.x + b.x, a.y + b.y};
96
+}
97
+
98
+inline __device__ float3 operator+(const float3 &a, const float3 &b) {
99
+ return {a.x + b.x, a.y + b.y, a.z + b.z};
100
+}
101
+
102
+inline __device__ float4 operator+(const float4 & a, const float4 & b){
103
+ return {a.x + b.x, a.y + b.y, a.z + b.z, a.w + b.w};
104
+}
105
+
106
+////////////////////////////////////////////////////////////////////////////////////////////////////
107
+
108
+template<int BYTES> struct BytesToType {};
109
+
110
+template<> struct BytesToType<16> {
111
+ using Type = uint4;
112
+ static_assert(sizeof(Type) == 16);
113
+};
114
+
115
+template<> struct BytesToType<8> {
116
+ using Type = uint64_t;
117
+ static_assert(sizeof(Type) == 8);
118
+};
119
+
120
+template<> struct BytesToType<4> {
121
+ using Type = uint32_t;
122
+ static_assert(sizeof(Type) == 4);
123
+};
124
+
125
+template<> struct BytesToType<2> {
126
+ using Type = uint16_t;
127
+ static_assert(sizeof(Type) == 2);
128
+};
129
+
130
+template<> struct BytesToType<1> {
131
+ using Type = uint8_t;
132
+ static_assert(sizeof(Type) == 1);
133
+};
134
+
135
+////////////////////////////////////////////////////////////////////////////////////////////////////
136
+
137
+template<typename scalar_t, int N>
138
+struct Converter{
139
+ static inline __device__ void to_float(const scalar_t (&src)[N], float (&dst)[N]) {
140
+ #pragma unroll
141
+ for (int i = 0; i < N; ++i) { dst[i] = src[i]; }
142
+ }
143
+};
144
+
145
+template<int N>
146
+struct Converter<at::Half, N>{
147
+ static inline __device__ void to_float(const at::Half (&src)[N], float (&dst)[N]) {
148
+ static_assert(N % 2 == 0);
149
+ auto &src2 = reinterpret_cast<const half2 (&)[N / 2]>(src);
150
+ auto &dst2 = reinterpret_cast<float2 (&)[N / 2]>(dst);
151
+ #pragma unroll
152
+ for (int i = 0; i < N / 2; ++i) { dst2[i] = __half22float2(src2[i]); }
153
+ }
154
+};
155
+
156
+#if __CUDA_ARCH__ >= 800
157
+template<int N>
158
+struct Converter<at::BFloat16, N>{
159
+ static inline __device__ void to_float(const at::BFloat16 (&src)[N], float (&dst)[N]) {
160
+ static_assert(N % 2 == 0);
161
+ auto &src2 = reinterpret_cast<const nv_bfloat162 (&)[N / 2]>(src);
162
+ auto &dst2 = reinterpret_cast<float2 (&)[N / 2]>(dst);
163
+ #pragma unroll
164
+ for (int i = 0; i < N / 2; ++i) { dst2[i] = __bfloat1622float2(src2[i]); }
165
+ }
166
+};
167
+#endif
168
+
169
+////////////////////////////////////////////////////////////////////////////////////////////////////
170
+
171
+
172
+template<typename scalar_t> struct SSMScanOp;
173
+
174
+template<>
175
+struct SSMScanOp<float> {
176
+ __device__ __forceinline__ float2 operator()(const float2 &ab0, const float2 &ab1) const {
177
+ return make_float2(ab1.x * ab0.x, ab1.x * ab0.y + ab1.y);
178
+ }
179
+};
180
+
181
+// A stateful callback functor that maintains a running prefix to be applied
182
+// during consecutive scan operations.
183
+template <typename scalar_t> struct SSMScanPrefixCallbackOp {
184
+ using scan_t = std::conditional_t<std::is_same_v<scalar_t, float>, float2, float4>;
185
+ scan_t running_prefix;
186
+ // Constructor
187
+ __device__ SSMScanPrefixCallbackOp(scan_t running_prefix_) : running_prefix(running_prefix_) {}
188
+ // Callback operator to be entered by the first warp of threads in the block.
189
+ // Thread-0 is responsible for returning a value for seeding the block-wide scan.
190
+ __device__ scan_t operator()(scan_t block_aggregate) {
191
+ scan_t old_prefix = running_prefix;
192
+ running_prefix = SSMScanOp<scalar_t>()(running_prefix, block_aggregate);
193
+ return old_prefix;
194
+ }
195
+};
196
+
197
+////////////////////////////////////////////////////////////////////////////////////////////////////
198
+
199
+template<typename Ktraits>
200
+inline __device__ void load_input(typename Ktraits::input_t *u,
201
+ typename Ktraits::input_t (&u_vals)[Ktraits::kNItems],
202
+ typename Ktraits::BlockLoadT::TempStorage &smem_load,
203
+ int seqlen) {
204
+ if constexpr (Ktraits::kIsEvenLen) {
205
+ auto& smem_load_vec = reinterpret_cast<typename Ktraits::BlockLoadVecT::TempStorage&>(smem_load);
206
+ using vec_t = typename Ktraits::vec_t;
207
+ typename Ktraits::BlockLoadVecT(smem_load_vec).Load(
208
+ reinterpret_cast<vec_t*>(u),
209
+ reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(u_vals)
210
+ #ifdef USE_ROCM
211
+ , Ktraits::kNThreads * Ktraits::kNLoads
212
+ #endif
213
+
214
+ );
215
+ } else {
216
+ typename Ktraits::BlockLoadT(smem_load).Load(u, u_vals, seqlen, 0.f);
217
+ }
218
+}
219
+
220
+template<typename Ktraits>
221
+inline __device__ void load_index(int *u,
222
+ int (&u_vals)[Ktraits::kNItems],
223
+ typename Ktraits::BlockLoadIndexT::TempStorage &smem_load_index,
224
+ int seqlen) {
225
+ if constexpr (Ktraits::kIsEvenLen) {
226
+ auto& smem_load_index_vec = reinterpret_cast<typename Ktraits::BlockLoadIndexVecT::TempStorage&>(smem_load_index);
227
+ Ktraits::BlockLoadIndexVecT(smem_load_index_vec).Load(
228
+ reinterpret_cast<uint4*>(u),
229
+ reinterpret_cast<uint4(&)[Ktraits::kNLoadsIndex]>(u_vals)
230
+ );
231
+ } else {
232
+ Ktraits::BlockLoadIndexT(smem_load_index).Load(u, u_vals, seqlen, 0);
233
+ }
234
+}
235
+
236
+template<typename Ktraits>
237
+inline __device__ void load_weight(typename Ktraits::input_t *Bvar,
238
+ typename Ktraits::weight_t (&B_vals)[Ktraits::kNItems],
239
+ typename Ktraits::BlockLoadWeightT::TempStorage &smem_load_weight,
240
+ int seqlen) {
241
+ constexpr int kNItems = Ktraits::kNItems;
242
+ typename Ktraits::input_t B_vals_load[kNItems];
243
+ if constexpr (Ktraits::kIsEvenLen) {
244
+ auto& smem_load_weight_vec = reinterpret_cast<typename Ktraits::BlockLoadWeightVecT::TempStorage&>(smem_load_weight);
245
+ using vec_t = typename Ktraits::vec_t;
246
+ typename Ktraits::BlockLoadWeightVecT(smem_load_weight_vec).Load(
247
+ reinterpret_cast<vec_t*>(Bvar),
248
+ reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(B_vals_load)
249
+ );
250
+ } else {
251
+ typename Ktraits::BlockLoadWeightT(smem_load_weight).Load(Bvar, B_vals_load, seqlen, 0.f);
252
+ }
253
+ // #pragma unroll
254
+ // for (int i = 0; i < kNItems; ++i) { B_vals[i] = B_vals_load[i]; }
255
+ Converter<typename Ktraits::input_t, kNItems>::to_float(B_vals_load, B_vals);
256
+}
257
+
258
+template<typename Ktraits>
259
+inline __device__ void store_output(typename Ktraits::input_t *out,
260
+ const float (&out_vals)[Ktraits::kNItems],
261
+ typename Ktraits::BlockStoreT::TempStorage &smem_store,
262
+ int seqlen) {
263
+ typename Ktraits::input_t write_vals[Ktraits::kNItems];
264
+ #pragma unroll
265
+ for (int i = 0; i < Ktraits::kNItems; ++i) { write_vals[i] = out_vals[i]; }
266
+ if constexpr (Ktraits::kIsEvenLen) {
267
+ auto& smem_store_vec = reinterpret_cast<typename Ktraits::BlockStoreVecT::TempStorage&>(smem_store);
268
+ using vec_t = typename Ktraits::vec_t;
269
+ typename Ktraits::BlockStoreVecT(smem_store_vec).Store(
270
+ reinterpret_cast<vec_t*>(out),
271
+ reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(write_vals)
272
+ );
273
+ } else {
274
+ typename Ktraits::BlockStoreT(smem_store).Store(out, write_vals, seqlen);
275
+ }
276
+}
csrc/mamba/mamba_ssm/selective_scan_fwd.cu
new
+593
@@ -0,0 +1,593 @@
1
+// clang-format off
2
+// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/selective_scan_fwd_kernel.cuh
3
+#include <torch/all.h>
4
+#include <ATen/cuda/CUDAContext.h>
5
+#include <c10/cuda/CUDAGuard.h>
6
+#include "selective_scan.h"
7
+
8
+#include <c10/util/BFloat16.h>
9
+#include <c10/util/Half.h>
10
+#include <c10/cuda/CUDAException.h> // For C10_CUDA_CHECK and C10_CUDA_KERNEL_LAUNCH_CHECK
11
+
12
+#ifndef USE_ROCM
13
+ #include <cub/block/block_load.cuh>
14
+ #include <cub/block/block_store.cuh>
15
+ #include <cub/block/block_scan.cuh>
16
+#else
17
+ #include <hipcub/hipcub.hpp>
18
+ namespace cub = hipcub;
19
+#endif
20
+
21
+#include "selective_scan.h"
22
+#include "static_switch.h"
23
+
24
+template<int kNThreads_, int kNItems_, int kNRows_, bool kIsEvenLen_,
25
+ bool kIsVariableB_, bool kIsVariableC_,
26
+ bool kHasZ_, bool kUseIndex_, typename input_t_, typename weight_t_>
27
+struct Selective_Scan_fwd_kernel_traits {
28
+ static_assert(kNItems_ % 4 == 0);
29
+ using input_t = input_t_;
30
+ using weight_t = weight_t_;
31
+ static constexpr int kNThreads = kNThreads_;
32
+ // Setting MinBlocksPerMP to be 3 (instead of 2) for 128 threads improves occupancy.
33
+ static constexpr int kMinBlocks = kNThreads < 128 ? 5 : 3;
34
+ static constexpr int kNItems = kNItems_;
35
+ static constexpr int kNRows = kNRows_;
36
+ static constexpr int kNBytes = sizeof(input_t);
37
+ static_assert(kNBytes == 2 || kNBytes == 4);
38
+ static constexpr int kNElts = kNBytes == 4 ? 4 : constexpr_min(8, kNItems);
39
+ static_assert(kNItems % kNElts == 0);
40
+ static constexpr int kNLoads = kNItems / kNElts;
41
+ static constexpr bool kIsEvenLen = kIsEvenLen_;
42
+ static constexpr bool kIsVariableB = kIsVariableB_;
43
+ static constexpr bool kIsVariableC = kIsVariableC_;
44
+ static constexpr bool kHasZ = kHasZ_;
45
+ static constexpr bool kUseIndex = kUseIndex_;
46
+
47
+ static constexpr bool kDirectIO = kIsEvenLen && kNLoads == 1;
48
+ static constexpr int kNLoadsIndex = kNItems / 4;
49
+ using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
50
+ using scan_t = float2;
51
+ using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
52
+ using BlockLoadVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads,
53
+ !kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
54
+ using BlockLoadIndexT = cub::BlockLoad<int, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
55
+ using BlockLoadIndexVecT = cub::BlockLoad<uint4, kNThreads, kNLoadsIndex,
56
+ !(kIsEvenLen && kNLoadsIndex == 1) ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
57
+ using BlockLoadWeightT = cub::BlockLoad<input_t, kNThreads, kNItems , cub::BLOCK_LOAD_WARP_TRANSPOSE>;
58
+ using BlockLoadWeightVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads ,
59
+ !kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
60
+ using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNItems, cub::BLOCK_STORE_WARP_TRANSPOSE>;
61
+ using BlockStoreVecT = cub::BlockStore<vec_t, kNThreads, kNLoads,
62
+ !kDirectIO ? cub::BLOCK_STORE_WARP_TRANSPOSE : cub::BLOCK_STORE_DIRECT>;
63
+ // using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_RAKING_MEMOIZE>;
64
+ // using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_RAKING>;
65
+ using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_WARP_SCANS>;
66
+ static constexpr int kSmemIOSize = custom_max({sizeof(typename BlockLoadT::TempStorage),
67
+ sizeof(typename BlockLoadVecT::TempStorage),
68
+ sizeof(typename BlockLoadIndexT::TempStorage),
69
+ sizeof(typename BlockLoadIndexVecT::TempStorage),
70
+ (int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightT::TempStorage),
71
+ (int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightVecT::TempStorage),
72
+ sizeof(typename BlockStoreT::TempStorage),
73
+ sizeof(typename BlockStoreVecT::TempStorage)});
74
+ static constexpr int kSmemSize = kSmemIOSize + sizeof(typename BlockScanT::TempStorage);
75
+};
76
+
77
+template<typename Ktraits>
78
+__global__ __launch_bounds__(Ktraits::kNThreads, Ktraits::kMinBlocks)
79
+void selective_scan_fwd_kernel(SSMParamsBase params) {
80
+ constexpr bool kIsVariableB = Ktraits::kIsVariableB;
81
+ constexpr bool kIsVariableC = Ktraits::kIsVariableC;
82
+ constexpr bool kHasZ = Ktraits::kHasZ;
83
+ constexpr bool kUseIndex = Ktraits::kUseIndex;
84
+ constexpr int kNThreads = Ktraits::kNThreads;
85
+ constexpr int kNItems = Ktraits::kNItems;
86
+ constexpr int kNRows = Ktraits::kNRows;
87
+ constexpr bool kDirectIO = Ktraits::kDirectIO;
88
+ using input_t = typename Ktraits::input_t;
89
+ using weight_t = typename Ktraits::weight_t;
90
+ using scan_t = typename Ktraits::scan_t;
91
+
92
+ // Shared memory.
93
+ extern __shared__ char smem_[];
94
+ // cast to lvalue reference of expected type
95
+ // char *smem_loadstorescan = smem_ + 2 * MAX_DSTATE * sizeof(weight_t);
96
+ // auto& smem_load = reinterpret_cast<typename BlockLoadT::TempStorage&>(smem_ + 2 * MAX_DSTATE * sizeof(weight_t));
97
+ // auto& smem_load = reinterpret_cast<typename BlockLoadT::TempStorage&>(smem_loadstorescan);
98
+ auto& smem_load = reinterpret_cast<typename Ktraits::BlockLoadT::TempStorage&>(smem_);
99
+ auto& smem_load_weight = reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage&>(smem_);
100
+ auto& smem_load_index = reinterpret_cast<typename Ktraits::BlockLoadIndexT::TempStorage&>(smem_);
101
+ auto& smem_load_weight1 = *reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage*>(smem_ + sizeof(typename Ktraits::BlockLoadWeightT::TempStorage));
102
+ auto& smem_store = reinterpret_cast<typename Ktraits::BlockStoreT::TempStorage&>(smem_);
103
+ auto& smem_scan = *reinterpret_cast<typename Ktraits::BlockScanT::TempStorage*>(smem_ + Ktraits::kSmemIOSize);
104
+ // weight_t *smem_a = reinterpret_cast<weight_t *>(smem_ + smem_loadstorescan_size);
105
+ // weight_t *smem_bc = reinterpret_cast<weight_t *>(smem_a + MAX_DSTATE);
106
+ scan_t *smem_running_prefix = reinterpret_cast<scan_t *>(smem_ + Ktraits::kSmemSize);
107
+
108
+ const int batch_id = blockIdx.x;
109
+ const int dim_id = blockIdx.y;
110
+ const int group_id = dim_id / (params.dim_ngroups_ratio);
111
+ input_t *u = reinterpret_cast<input_t *>(params.u_ptr) + batch_id * params.u_batch_stride
112
+ + dim_id * kNRows * params.u_d_stride;
113
+ input_t *delta = reinterpret_cast<input_t *>(params.delta_ptr) + batch_id * params.delta_batch_stride
114
+ + dim_id * kNRows * params.delta_d_stride;
115
+ weight_t *A = reinterpret_cast<weight_t *>(params.A_ptr) + dim_id * kNRows * params.A_d_stride;
116
+ weight_t *B = reinterpret_cast<weight_t *>(params.B_ptr) + dim_id * kNRows * params.B_d_stride;
117
+ input_t *Bvar = reinterpret_cast<input_t *>(params.B_ptr) + batch_id * params.B_batch_stride + group_id * params.B_group_stride;
118
+ weight_t *C = reinterpret_cast<weight_t *>(params.C_ptr) + dim_id * kNRows * params.C_d_stride;
119
+ input_t *Cvar = reinterpret_cast<input_t *>(params.C_ptr) + batch_id * params.C_batch_stride + group_id * params.C_group_stride;
120
+ scan_t *x = reinterpret_cast<scan_t *>(params.x_ptr) + (batch_id * params.dim + dim_id * kNRows) * params.n_chunks * params.dstate;
121
+ int *index = !kUseIndex ? nullptr :reinterpret_cast<int *>(params.index_ptr) + batch_id * params.seqlen;
122
+
123
+ float D_val[kNRows] = {0};
124
+ if (params.D_ptr != nullptr) {
125
+ #pragma unroll
126
+ for (int r = 0; r < kNRows; ++r) {
127
+ D_val[r] = reinterpret_cast<float *>(params.D_ptr)[dim_id * kNRows + r];
128
+ }
129
+ }
130
+ float delta_bias[kNRows] = {0};
131
+ if (params.delta_bias_ptr != nullptr) {
132
+ #pragma unroll
133
+ for (int r = 0; r < kNRows; ++r) {
134
+ delta_bias[r] = reinterpret_cast<float *>(params.delta_bias_ptr)[dim_id * kNRows + r];
135
+ }
136
+ }
137
+
138
+
139
+ // for (int state_idx = threadIdx.x; state_idx < params.dstate; state_idx += blockDim.x) {
140
+ // smem_a[state_idx] = A[state_idx * params.A_dstate_stride];
141
+ // smem_bc[state_idx] = B[state_idx * params.B_dstate_stride] * C[state_idx * params.C_dstate_stride];
142
+ // }
143
+
144
+ constexpr int kChunkSize = kNThreads * kNItems;
145
+ for (int chunk = 0; chunk < params.n_chunks; ++chunk) {
146
+ input_t u_vals[kNRows][kNItems], delta_vals_load[kNRows][kNItems];
147
+ int index_vals_load[kNRows][kNItems];
148
+
149
+ __syncthreads();
150
+ #pragma unroll
151
+ for (int r = 0; r < kNRows; ++r) {
152
+ if constexpr (!kDirectIO) {
153
+ if (r > 0) { __syncthreads(); }
154
+ }
155
+ load_input<Ktraits>(u + r * params.u_d_stride, u_vals[r], smem_load, params.seqlen - chunk * kChunkSize);
156
+ if constexpr (!kDirectIO) { __syncthreads(); }
157
+ load_input<Ktraits>(delta + r * params.delta_d_stride, delta_vals_load[r], smem_load, params.seqlen - chunk * kChunkSize);
158
+ if constexpr (kUseIndex) {
159
+ load_index<Ktraits>(index + r * params.delta_d_stride, index_vals_load[r], smem_load_index, params.seqlen - chunk * kChunkSize);
160
+ }
161
+ }
162
+ if constexpr (kUseIndex) {
163
+ index += kChunkSize;
164
+ }
165
+ u += kChunkSize;
166
+ delta += kChunkSize;
167
+
168
+ float delta_vals[kNRows][kNItems], delta_u_vals[kNRows][kNItems], out_vals[kNRows][kNItems];
169
+ #pragma unroll
170
+ for (int r = 0; r < kNRows; ++r) {
171
+ #pragma unroll
172
+ for (int i = 0; i < kNItems; ++i) {
173
+ float u_val = float(u_vals[r][i]);
174
+ delta_vals[r][i] = float(delta_vals_load[r][i]) + delta_bias[r];
175
+ if (params.delta_softplus) {
176
+ delta_vals[r][i] = delta_vals[r][i] <= 20.f ? log1pf(expf(delta_vals[r][i])) : delta_vals[r][i];
177
+ }
178
+ delta_u_vals[r][i] = delta_vals[r][i] * u_val;
179
+ out_vals[r][i] = D_val[r] * u_val;
180
+ }
181
+ }
182
+
183
+ __syncthreads();
184
+ for (int state_idx = 0; state_idx < params.dstate; ++state_idx) {
185
+ weight_t A_val[kNRows];
186
+ #pragma unroll
187
+ for (int r = 0; r < kNRows; ++r) {
188
+ A_val[r] = A[state_idx * params.A_dstate_stride + r * params.A_d_stride];
189
+ // Multiply the real part of A with LOG2E so we can use exp2f instead of expf.
190
+ constexpr float kLog2e = M_LOG2E;
191
+ A_val[r] *= kLog2e;
192
+ }
193
+ // This variable holds B * C if both B and C are constant across seqlen. If only B varies
194
+ // across seqlen, this holds C. If only C varies across seqlen, this holds B.
195
+ // If both B and C vary, this is unused.
196
+ weight_t BC_val[kNRows];
197
+ weight_t B_vals[kNItems], C_vals[kNItems];
198
+ if constexpr (kIsVariableB) {
199
+ load_weight<Ktraits>(Bvar + state_idx * params.B_dstate_stride, B_vals,
200
+ smem_load_weight, (params.seqlen - chunk * kChunkSize) * (1));
201
+ if constexpr (!kIsVariableC) {
202
+ #pragma unroll
203
+ for (int r = 0; r < kNRows; ++r) {
204
+ BC_val[r] = C[state_idx * params.C_dstate_stride + r * params.C_d_stride];
205
+ }
206
+ }
207
+ }
208
+ if constexpr (kIsVariableC) {
209
+ auto &smem_load_weight_C = !kIsVariableB ? smem_load_weight : smem_load_weight1;
210
+ load_weight<Ktraits>(Cvar + state_idx * params.C_dstate_stride, C_vals,
211
+ smem_load_weight_C, (params.seqlen - chunk * kChunkSize) * (1 ));
212
+ if constexpr (!kIsVariableB) {
213
+ #pragma unroll
214
+ for (int r = 0; r < kNRows; ++r) {
215
+ BC_val[r] = B[state_idx * params.B_dstate_stride + r * params.B_d_stride];
216
+ }
217
+ }
218
+ }
219
+ if constexpr (!kIsVariableB && !kIsVariableC) {
220
+ #pragma unroll
221
+ for (int r = 0; r < kNRows; ++r) {
222
+ BC_val[r] = B[state_idx * params.B_dstate_stride + r * params.B_d_stride] * C[state_idx * params.C_dstate_stride + r * params.C_d_stride];
223
+ }
224
+ }
225
+
226
+ #pragma unroll
227
+ for (int r = 0; r < kNRows; ++r) {
228
+ if (r > 0) { __syncthreads(); } // Scan could be using the same smem
229
+ scan_t thread_data[kNItems];
230
+ #pragma unroll
231
+ for (int i = 0; i < kNItems; ++i) {
232
+ thread_data[i] = make_float2(exp2f(delta_vals[r][i] * A_val[r]),
233
+ !kIsVariableB ? delta_u_vals[r][i] : B_vals[i] * delta_u_vals[r][i]);
234
+
235
+ // Reset A bar for cumulative sequences (Real)
236
+ if constexpr (kUseIndex) {
237
+ if (index_vals_load[r][i] == 0) {
238
+ thread_data[i].x = 0.f;
239
+ }
240
+ }
241
+
242
+ if constexpr (!Ktraits::kIsEvenLen) { // So that the last state is correct
243
+ if (threadIdx.x * kNItems + i >= params.seqlen - chunk * kChunkSize) {
244
+ thread_data[i] = make_float2(1.f, 0.f);
245
+ }
246
+ }
247
+ }
248
+ // Initialize running total
249
+ scan_t running_prefix;
250
+ // If we use WARP_SCAN then all lane 0 of all warps (not just thread 0) needs to read
251
+ running_prefix = chunk == 0 ? x[(r * params.n_chunks) * params.dstate + state_idx] : ( threadIdx.x % 32 == 0 ? smem_running_prefix[state_idx + r * MAX_DSTATE] : make_float2(1.f, 0.f));
252
+ // running_prefix = chunk > 0 && threadIdx.x == 0 ? smem_running_prefix[state_idx] : make_float2(1.f, 0.f);
253
+ SSMScanPrefixCallbackOp<weight_t> prefix_op(running_prefix);
254
+ typename Ktraits::BlockScanT(smem_scan).InclusiveScan(
255
+ thread_data, thread_data, SSMScanOp<weight_t>(), prefix_op
256
+ );
257
+ // There's a syncthreads in the scan op, so we don't need to sync here.
258
+ // Unless there's only 1 warp, but then it's the same thread (0) reading and writing.
259
+ if (threadIdx.x == 0) {
260
+ smem_running_prefix[state_idx] = prefix_op.running_prefix;
261
+ x[(r * params.n_chunks + chunk) * params.dstate + state_idx] = prefix_op.running_prefix;
262
+ }
263
+ #pragma unroll
264
+ for (int i = 0; i < kNItems; ++i) {
265
+ const weight_t C_val = !kIsVariableC
266
+ ? BC_val[r]
267
+ : (!kIsVariableB ? BC_val[r] * C_vals[i] : C_vals[i]);
268
+ out_vals[r][i] += thread_data[i].y * C_val;
269
+ }
270
+ }
271
+ }
272
+
273
+ input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
274
+ + dim_id * kNRows * params.out_d_stride + chunk * kChunkSize;
275
+ __syncthreads();
276
+ #pragma unroll
277
+ for (int r = 0; r < kNRows; ++r) {
278
+ if constexpr (!kDirectIO) {
279
+ if (r > 0) { __syncthreads(); }
280
+ }
281
+ store_output<Ktraits>(out + r * params.out_d_stride, out_vals[r], smem_store, params.seqlen - chunk * kChunkSize);
282
+ }
283
+
284
+ if constexpr (kHasZ) {
285
+ input_t *z = reinterpret_cast<input_t *>(params.z_ptr) + batch_id * params.z_batch_stride
286
+ + dim_id * kNRows * params.z_d_stride + chunk * kChunkSize;
287
+ input_t *out_z = reinterpret_cast<input_t *>(params.out_z_ptr) + batch_id * params.out_z_batch_stride
288
+ + dim_id * kNRows * params.out_z_d_stride + chunk * kChunkSize;
289
+ #pragma unroll
290
+ for (int r = 0; r < kNRows; ++r) {
291
+ input_t z_vals[kNItems];
292
+ __syncthreads();
293
+ load_input<Ktraits>(z + r * params.z_d_stride, z_vals, smem_load, params.seqlen - chunk * kChunkSize);
294
+ #pragma unroll
295
+ for (int i = 0; i < kNItems; ++i) {
296
+ float z_val = z_vals[i];
297
+ out_vals[r][i] *= z_val / (1 + expf(-z_val));
298
+ }
299
+ __syncthreads();
300
+ store_output<Ktraits>(out_z + r * params.out_z_d_stride, out_vals[r], smem_store, params.seqlen - chunk * kChunkSize);
301
+ }
302
+ }
303
+
304
+ Bvar += kChunkSize * 1;
305
+ Cvar += kChunkSize * 1;
306
+ }
307
+}
308
+
309
+template<int kNThreads, int kNItems, typename input_t, typename weight_t>
310
+void selective_scan_fwd_launch(SSMParamsBase ¶ms, cudaStream_t stream) {
311
+ // Only kNRows == 1 is tested for now, which ofc doesn't differ from previously when we had each block
312
+ // processing 1 row.
313
+ constexpr int kNRows = 1;
314
+ // kIsVariableB, kIsVariableC and kHasZ are all set to True to reduce binary size
315
+ constexpr bool kIsVariableB = true;
316
+ constexpr bool kIsVariableC = true;
317
+ constexpr bool kHasZ = true;
318
+ BOOL_SWITCH(params.seqlen % (kNThreads * kNItems) == 0, kIsEvenLen, [&] {
319
+ BOOL_SWITCH(params.index_ptr != nullptr , kUseIndex, [&] {
320
+ using Ktraits = Selective_Scan_fwd_kernel_traits<kNThreads, kNItems, kNRows, kIsEvenLen, kIsVariableB, kIsVariableC, kHasZ, kUseIndex, input_t, weight_t>;
321
+ constexpr int kSmemSize = Ktraits::kSmemSize + kNRows * MAX_DSTATE * sizeof(typename Ktraits::scan_t);
322
+ dim3 grid(params.batch, params.dim / kNRows);
323
+ auto kernel = &selective_scan_fwd_kernel<Ktraits>;
324
+ if (kSmemSize >= 48 * 1024) {
325
+ C10_CUDA_CHECK(cudaFuncSetAttribute(
326
+ kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
327
+ }
328
+ kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
329
+ C10_CUDA_KERNEL_LAUNCH_CHECK();
330
+ });
331
+ });
332
+}
333
+
334
+template<typename input_t, typename weight_t>
335
+void selective_scan_fwd_cuda(SSMParamsBase ¶ms, cudaStream_t stream) {
336
+
337
+ #ifndef USE_ROCM
338
+ if (params.seqlen <= 128) {
339
+ selective_scan_fwd_launch<32, 4, input_t, weight_t>(params, stream);
340
+ } else if (params.seqlen <= 256) {
341
+ selective_scan_fwd_launch<32, 8, input_t, weight_t>(params, stream);
342
+ } else if (params.seqlen <= 512) {
343
+ selective_scan_fwd_launch<32, 16, input_t, weight_t>(params, stream);
344
+ } else if (params.seqlen <= 1024) {
345
+ selective_scan_fwd_launch<64, 16, input_t, weight_t>(params, stream);
346
+ } else {
347
+ selective_scan_fwd_launch<128, 16, input_t, weight_t>(params, stream);
348
+ }
349
+ #else
350
+ if (params.seqlen <= 256) {
351
+ selective_scan_fwd_launch<64, 4, input_t, weight_t>(params, stream);
352
+ } else if (params.seqlen <= 512) {
353
+ selective_scan_fwd_launch<64, 8, input_t, weight_t>(params, stream);
354
+ } else if (params.seqlen <= 1024) {
355
+ selective_scan_fwd_launch<64, 16, input_t, weight_t>(params, stream);
356
+ } else {
357
+ selective_scan_fwd_launch<128, 16, input_t, weight_t>(params, stream);
358
+ }
359
+ #endif
360
+}
361
+
362
+template void selective_scan_fwd_cuda<at::BFloat16, float>(SSMParamsBase ¶ms, cudaStream_t stream);
363
+template void selective_scan_fwd_cuda<at::Half, float>(SSMParamsBase ¶ms, cudaStream_t stream);
364
+template void selective_scan_fwd_cuda<float, float>(SSMParamsBase ¶ms, cudaStream_t stream);
365
+
366
+#define CHECK_SHAPE(x, ...) TORCH_CHECK(x.sizes() == torch::IntArrayRef({__VA_ARGS__}), #x " must have shape (" #__VA_ARGS__ ")")
367
+
368
+#define DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(ITYPE, NAME, ...) \
369
+ if (ITYPE == at::ScalarType::Half) { \
370
+ using input_t = at::Half; \
371
+ using weight_t = float; \
372
+ __VA_ARGS__(); \
373
+ } else if (ITYPE == at::ScalarType::BFloat16) { \
374
+ using input_t = at::BFloat16; \
375
+ using weight_t = float; \
376
+ __VA_ARGS__(); \
377
+ } else if (ITYPE == at::ScalarType::Float) { \
378
+ using input_t = float; \
379
+ using weight_t = float; \
380
+ __VA_ARGS__(); \
381
+ } else { \
382
+ AT_ERROR(#NAME, " not implemented for input type '", toString(ITYPE), "'"); \
383
+ }
384
+
385
+
386
+template<typename input_t, typename weight_t>
387
+void selective_scan_fwd_cuda(SSMParamsBase ¶ms, cudaStream_t stream);
388
+
389
+void set_ssm_params_fwd(SSMParamsBase ¶ms,
390
+ // sizes
391
+ const size_t batch,
392
+ const size_t dim,
393
+ const size_t seqlen,
394
+ const size_t dstate,
395
+ const size_t n_groups,
396
+ const size_t n_chunks,
397
+ const bool is_variable_B,
398
+ const bool is_variable_C,
399
+ // device pointers
400
+ const torch::Tensor u,
401
+ const torch::Tensor delta,
402
+ const torch::Tensor A,
403
+ const torch::Tensor B,
404
+ const torch::Tensor C,
405
+ const torch::Tensor out,
406
+ const torch::Tensor z,
407
+ const torch::Tensor out_z,
408
+ void* D_ptr,
409
+ void* delta_bias_ptr,
410
+ void* x_ptr,
411
+ bool has_z,
412
+ bool delta_softplus,
413
+ void* index_ptr) {
414
+
415
+ // Reset the parameters
416
+ memset(¶ms, 0, sizeof(params));
417
+
418
+ params.batch = batch;
419
+ params.dim = dim;
420
+ params.seqlen = seqlen;
421
+ params.dstate = dstate;
422
+ params.n_groups = n_groups;
423
+ params.n_chunks = n_chunks;
424
+ params.dim_ngroups_ratio = dim / n_groups;
425
+
426
+ params.delta_softplus = delta_softplus;
427
+
428
+ params.is_variable_B = is_variable_B;
429
+ params.is_variable_C = is_variable_C;
430
+
431
+ // Set the pointers and strides.
432
+ params.u_ptr = u.data_ptr();
433
+ params.delta_ptr = delta.data_ptr();
434
+ params.A_ptr = A.data_ptr();
435
+ params.B_ptr = B.data_ptr();
436
+ params.C_ptr = C.data_ptr();
437
+ params.D_ptr = D_ptr;
438
+ params.delta_bias_ptr = delta_bias_ptr;
439
+ params.out_ptr = out.data_ptr();
440
+ params.x_ptr = x_ptr;
441
+ params.z_ptr = has_z ? z.data_ptr() : nullptr;
442
+ params.out_z_ptr = has_z ? out_z.data_ptr() : nullptr;
443
+
444
+ params.index_ptr = index_ptr;
445
+
446
+ // All stride are in elements, not bytes.
447
+ params.A_d_stride = A.stride(0);
448
+ params.A_dstate_stride = A.stride(1);
449
+ if (!is_variable_B) {
450
+ params.B_d_stride = B.stride(0);
451
+ } else {
452
+ params.B_batch_stride = B.stride(0);
453
+ params.B_group_stride = B.stride(1);
454
+ }
455
+ params.B_dstate_stride = !is_variable_B ? B.stride(1) : B.stride(2);
456
+ if (!is_variable_C) {
457
+ params.C_d_stride = C.stride(0);
458
+ } else {
459
+ params.C_batch_stride = C.stride(0);
460
+ params.C_group_stride = C.stride(1);
461
+ }
462
+ params.C_dstate_stride = !is_variable_C ? C.stride(1) : C.stride(2);
463
+ params.u_batch_stride = u.stride(0);
464
+ params.u_d_stride = u.stride(1);
465
+ params.delta_batch_stride = delta.stride(0);
466
+ params.delta_d_stride = delta.stride(1);
467
+ if (has_z) {
468
+ params.z_batch_stride = z.stride(0);
469
+ params.z_d_stride = z.stride(1);
470
+ params.out_z_batch_stride = out_z.stride(0);
471
+ params.out_z_d_stride = out_z.stride(1);
472
+ }
473
+ params.out_batch_stride = out.stride(0);
474
+ params.out_d_stride = out.stride(1);
475
+}
476
+
477
+std::vector<torch::Tensor>
478
+selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
479
+ const torch::Tensor &A, const torch::Tensor &B, const torch::Tensor &C,
480
+ const c10::optional<torch::Tensor> &D_,
481
+ const c10::optional<torch::Tensor> &z_,
482
+ const c10::optional<torch::Tensor> &delta_bias_,
483
+ bool delta_softplus,
484
+ const c10::optional<torch::Tensor> &index_,
485
+ const c10::optional<torch::Tensor> &x) {
486
+ auto input_type = u.scalar_type();
487
+ auto weight_type = A.scalar_type();
488
+ TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
489
+ TORCH_CHECK(weight_type == at::ScalarType::Float);
490
+
491
+ const bool is_variable_B = B.dim() >= 3;
492
+ const bool is_variable_C = C.dim() >= 3;
493
+
494
+ TORCH_CHECK(delta.scalar_type() == input_type);
495
+ TORCH_CHECK(B.scalar_type() == (!is_variable_B ? weight_type : input_type));
496
+ TORCH_CHECK(C.scalar_type() == (!is_variable_C ? weight_type : input_type));
497
+
498
+ TORCH_CHECK(u.is_cuda());
499
+ TORCH_CHECK(delta.is_cuda());
500
+ TORCH_CHECK(A.is_cuda());
501
+ TORCH_CHECK(B.is_cuda());
502
+ TORCH_CHECK(C.is_cuda());
503
+
504
+ TORCH_CHECK(u.stride(-1) == 1 || u.size(-1) == 1);
505
+ TORCH_CHECK(delta.stride(-1) == 1 || delta.size(-1) == 1);
506
+
507
+ const auto sizes = u.sizes();
508
+ const int batch_size = sizes[0];
509
+ const int dim = sizes[1];
510
+ const int seqlen = sizes[2];
511
+ const int dstate = A.size(1);
512
+ const int n_groups = is_variable_B ? B.size(1) : 1;
513
+
514
+ TORCH_CHECK(dstate <= 256, "selective_scan only supports state dimension <= 256");
515
+
516
+ CHECK_SHAPE(u, batch_size, dim, seqlen);
517
+ CHECK_SHAPE(delta, batch_size, dim, seqlen);
518
+ CHECK_SHAPE(A, dim, dstate);
519
+ TORCH_CHECK(is_variable_B, "is_variable_B = False is disabled in favor of reduced binary size")
520
+ CHECK_SHAPE(B, batch_size, n_groups, dstate, seqlen );
521
+ TORCH_CHECK(B.stride(-1) == 1 || B.size(-1) == 1);
522
+
523
+ TORCH_CHECK(is_variable_C, "is_variable_C = False is disabled in favor of reduced binary size")
524
+ CHECK_SHAPE(C, batch_size, n_groups, dstate, seqlen);
525
+ TORCH_CHECK(C.stride(-1) == 1 || C.size(-1) == 1);
526
+
527
+ if (D_.has_value()) {
528
+ auto D = D_.value();
529
+ TORCH_CHECK(D.scalar_type() == at::ScalarType::Float);
530
+ TORCH_CHECK(D.is_cuda());
531
+ TORCH_CHECK(D.stride(-1) == 1 || D.size(-1) == 1);
532
+ CHECK_SHAPE(D, dim);
533
+ }
534
+
535
+ if (delta_bias_.has_value()) {
536
+ auto delta_bias = delta_bias_.value();
537
+ TORCH_CHECK(delta_bias.scalar_type() == at::ScalarType::Float);
538
+ TORCH_CHECK(delta_bias.is_cuda());
539
+ TORCH_CHECK(delta_bias.stride(-1) == 1 || delta_bias.size(-1) == 1);
540
+ CHECK_SHAPE(delta_bias, dim);
541
+ }
542
+ if (index_.has_value()) {
543
+ auto index = index_.value();
544
+ TORCH_CHECK(index.scalar_type() == at::ScalarType::Int);
545
+ TORCH_CHECK(index.is_cuda());
546
+ CHECK_SHAPE(index, batch_size, seqlen);
547
+ }
548
+
549
+ at::Tensor z, out_z;
550
+ const bool has_z = z_.has_value();
551
+ TORCH_CHECK(has_z, "has_z = False is disabled in favor of reduced binary size")
552
+ z = z_.value();
553
+ TORCH_CHECK(z.scalar_type() == input_type);
554
+ TORCH_CHECK(z.is_cuda());
555
+ TORCH_CHECK(z.stride(-1) == 1 || z.size(-1) == 1);
556
+ CHECK_SHAPE(z, batch_size, dim, seqlen);
557
+ out_z = torch::empty_like(z);
558
+
559
+ const int n_chunks = (seqlen + 2048 - 1) / 2048;
560
+ // const int n_chunks = (seqlen + 1024 - 1) / 1024;
561
+ // at::Tensor out = torch::empty_like(u);
562
+ // Right now u has BHL layout and delta has HBL layout, and we want out to have HBL layout
563
+ at::Tensor out = torch::empty_like(delta);
564
+ if (x.has_value()){
565
+ auto _x = x.value();
566
+ TORCH_CHECK(_x.scalar_type() == weight_type);
567
+ TORCH_CHECK(_x.is_cuda());
568
+ TORCH_CHECK(_x.stride(-1) == 1);
569
+ CHECK_SHAPE(_x, batch_size, dim, n_chunks, dstate * 2);
570
+ }
571
+
572
+ SSMParamsBase params;
573
+ set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, n_chunks, is_variable_B, is_variable_C,
574
+ u, delta, A, B, C, out, z, out_z,
575
+ D_.has_value() ? D_.value().data_ptr() : nullptr,
576
+ delta_bias_.has_value() ? delta_bias_.value().data_ptr() : nullptr,
577
+ x.value().data_ptr(),
578
+ has_z,
579
+ delta_softplus,
580
+ index_.has_value() ? index_.value().data_ptr() : nullptr);
581
+
582
+ // Otherwise the kernel will be launched from cuda:0 device
583
+ // Cast to char to avoid compiler warning about narrowing
584
+ at::cuda::CUDAGuard device_guard{(char)u.get_device()};
585
+ auto stream = at::cuda::getCurrentCUDAStream().stream();
586
+ DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(u.scalar_type(), "selective_scan_fwd", [&] {
587
+ selective_scan_fwd_cuda<input_t, weight_t>(params, stream);
588
+ });
589
+ std::vector<at::Tensor> result = {out, x.value()};
590
+ if (has_z) { result.push_back(out_z); }
591
+ return result;
592
+}
593
+
csrc/mamba/mamba_ssm/static_switch.h
new
+28
@@ -0,0 +1,28 @@
1
+// Inspired by
2
+// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h
3
+// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h
4
+
5
+// clang-format off
6
+// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/static_switch.h
7
+#pragma once
8
+
9
+/// @param COND - a boolean expression to switch by
10
+/// @param CONST_NAME - a name given for the constexpr bool variable.
11
+/// @param ... - code to execute for true and false
12
+///
13
+/// Usage:
14
+/// ```
15
+/// BOOL_SWITCH(flag, BoolConst, [&] {
16
+/// some_function<BoolConst>(...);
17
+/// });
18
+/// ```
19
+#define BOOL_SWITCH(COND, CONST_NAME, ...) \
20
+ [&] { \
21
+ if (COND) { \
22
+ constexpr bool CONST_NAME = true; \
23
+ return __VA_ARGS__(); \
24
+ } else { \
25
+ constexpr bool CONST_NAME = false; \
26
+ return __VA_ARGS__(); \
27
+ } \
28
+ }()
csrc/ops.h
+22
@@ -195,6 +195,28 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
195
torch::Tensor experts_ids,
196
torch::Tensor num_tokens_post_pad);
197
198
+std::vector<torch::Tensor> selective_scan_fwd(
199
+ const torch::Tensor& u, const torch::Tensor& delta, const torch::Tensor& A,
200
+ const torch::Tensor& B, const torch::Tensor& C,
201
+ const c10::optional<torch::Tensor>& D_,
202
+ const c10::optional<torch::Tensor>& z_,
203
+ const c10::optional<torch::Tensor>& delta_bias_, bool delta_softplus,
204
+ const c10::optional<torch::Tensor>& index_,
205
+ const c10::optional<torch::Tensor>& x);
206
+
207
+at::Tensor causal_conv1d_update(const at::Tensor& x,
208
+ const at::Tensor& conv_state,
209
+ const at::Tensor& weight,
210
+ const c10::optional<at::Tensor>& bias_,
211
+ bool silu_activation);
212
+
213
+at::Tensor causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
214
+ const c10::optional<at::Tensor>& bias_,
215
+ const c10::optional<at::Tensor>& seq_idx_,
216
+ const c10::optional<at::Tensor>& initial_states_,
217
+ const c10::optional<at::Tensor>& final_states_out_,
218
+ bool silu_activation);
219
+
220
#ifndef USE_ROCM
221
using fptr_t = int64_t;
222
fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
csrc/torch_bindings.cpp
+25
@@ -202,6 +202,31 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
202
ops.def("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
203
ops.impl("cutlass_scaled_mm_supports_fp8", torch::kCUDA,
204
&cutlass_scaled_mm_supports_fp8);
205
+ // Mamba selective scan kernel
206
+ ops.def(
207
+ "selective_scan_fwd(Tensor! u, Tensor! delta,"
208
+ "Tensor! A, Tensor! B, Tensor! C,"
209
+ "Tensor? D_, Tensor? z_, Tensor? delta_bias_,"
210
+ "bool delta_softplus,"
211
+ "Tensor? index_, Tensor? x) -> Tensor[]");
212
+ ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
213
+
214
+ ops.def(
215
+ "causal_conv1d_update(Tensor! x,"
216
+ "Tensor! conv_state,"
217
+ "Tensor! weight,"
218
+ "Tensor? bias_,"
219
+ "bool silu_activation) -> Tensor");
220
+ ops.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update);
221
+
222
+ ops.def(
223
+ "causal_conv1d_fwd(Tensor! x, Tensor! weight,"
224
+ "Tensor? bias_,"
225
+ "Tensor? seq_idx_,"
226
+ "Tensor? initial_states_,"
227
+ "Tensor? final_states_out_,"
228
+ "bool silu_activation) -> Tensor");
229
+ ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
230
#endif
231
232
// Quantized GEMM for GPTQ.
requirements-mamba.txt
deleted
-3
@@ -1,3 +0,0 @@
1
-# Mamba dependencies
2
-mamba-ssm>=1.2.2
3
-causal-conv1d>=1.2.0
requirements-test.txt
+1
-1
@@ -11,7 +11,7 @@ pytest-shard
11
12
# testing utils
13
awscli
14
-einops # required for MPT and qwen-vl
14
+einops # required for MPT, qwen-vl and Mamba
15
httpx
16
peft
17
requests
tests/kernels/test_causal_conv1d.py
new
+205
@@ -0,0 +1,205 @@
1
+from typing import Optional
2
+
3
+import pytest
4
+import torch
5
+import torch.nn.functional as F
6
+from einops import rearrange
7
+
8
+from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
9
+ causal_conv1d_fn, causal_conv1d_update)
10
+
11
+
12
+def causal_conv1d_ref(
13
+ x: torch.Tensor,
14
+ weight: torch.Tensor,
15
+ bias: Optional[torch.Tensor] = None,
16
+ initial_states: Optional[torch.Tensor] = None,
17
+ return_final_states: bool = False,
18
+ final_states_out: Optional[torch.Tensor] = None,
19
+ activation: Optional[str] = "silu",
20
+):
21
+ """
22
+ x: (batch, dim, seqlen)
23
+ weight: (dim, width)
24
+ bias: (dim,)
25
+ initial_states: (batch, dim, width - 1)
26
+ final_states_out: (batch, dim, width - 1)
27
+
28
+ out: (batch, dim, seqlen)
29
+ """
30
+ if activation not in [None, "silu", "swish"]:
31
+ raise NotImplementedError("activation must be None, silu, or swish")
32
+ dtype_in = x.dtype
33
+ x = x.to(weight.dtype)
34
+ seqlen = x.shape[-1]
35
+ dim, width = weight.shape
36
+ if initial_states is None:
37
+ out = F.conv1d(x,
38
+ weight.unsqueeze(1),
39
+ bias,
40
+ padding=width - 1,
41
+ groups=dim)
42
+ else:
43
+ x = torch.cat([initial_states, x], dim=-1)
44
+ out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim)
45
+ out = out[..., :seqlen]
46
+ if return_final_states:
47
+ final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to(
48
+ dtype_in) # (batch, dim, width - 1)
49
+ if final_states_out is not None:
50
+ final_states_out.copy_(final_states)
51
+ else:
52
+ final_states_out = final_states
53
+ out = (out if activation is None else F.silu(out)).to(dtype=dtype_in)
54
+ return (out, None) if not return_final_states else (out, final_states_out)
55
+
56
+
57
+def causal_conv1d_update_ref(x: torch.Tensor,
58
+ conv_state: torch.Tensor,
59
+ weight: torch.Tensor,
60
+ bias: Optional[torch.Tensor] = None,
61
+ activation: Optional[str] = None):
62
+ """
63
+ x: (batch, dim)
64
+ conv_state: (batch, dim, width)
65
+ weight: (dim, width)
66
+ bias: (dim,)
67
+
68
+ out: (batch, dim)
69
+ """
70
+ if activation not in [None, "silu", "swish"]:
71
+ raise NotImplementedError("activation must be None, silu, or swish")
72
+ dtype_in = x.dtype
73
+ batch, dim = x.shape
74
+ width = weight.shape[1]
75
+ assert conv_state.shape == (batch, dim, width)
76
+ assert weight.shape == (dim, width)
77
+ conv_state.copy_(torch.roll(conv_state, shifts=-1,
78
+ dims=-1)) # Update state (B D W)
79
+ conv_state[:, :, -1] = x
80
+ out = torch.sum(conv_state * weight, dim=-1) # (B D)
81
+ if bias is not None:
82
+ out += bias
83
+ return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
84
+
85
+
86
+@pytest.mark.parametrize("return_final_states", [False, True])
87
+@pytest.mark.parametrize("has_initial_states", [False, True])
88
+@pytest.mark.parametrize("channel_last", [False, True])
89
+@pytest.mark.parametrize("itype", [torch.bfloat16])
90
+@pytest.mark.parametrize("silu_activation", [False, True])
91
+@pytest.mark.parametrize("has_bias", [False, True])
92
+@pytest.mark.parametrize("width", [4])
93
+@pytest.mark.parametrize("seqlen", [128, 512, 4096])
94
+@pytest.mark.parametrize('dim', [64, 4096 + 32])
95
+@pytest.mark.parametrize('batch', [1, 2])
96
+def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
97
+ itype, channel_last, has_initial_states,
98
+ return_final_states):
99
+ if not channel_last and (has_initial_states or return_final_states):
100
+ pytest.skip(
101
+ "Only channel_last support initial_states or return_final_states")
102
+ device = "cuda"
103
+ rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
104
+ if itype == torch.bfloat16:
105
+ rtol, atol = 1e-2, 5e-2
106
+ # set seed
107
+ torch.random.manual_seed(0)
108
+ if not channel_last:
109
+ x = torch.randn(batch,
110
+ 4096 + dim + 64,
111
+ seqlen,
112
+ device=device,
113
+ dtype=itype)[:, 4096:4096 + dim, :]
114
+ else:
115
+ x = rearrange(
116
+ torch.randn(batch,
117
+ seqlen,
118
+ 4096 + dim + 64,
119
+ device=device,
120
+ dtype=itype)[:, :, 4096:4096 + dim], "b s d -> b d s")
121
+ weight = torch.randn(dim, width, device=device, dtype=itype)
122
+ bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
123
+ if has_initial_states:
124
+ initial_states = torch.randn(batch,
125
+ width - 1,
126
+ dim,
127
+ device=device,
128
+ dtype=itype).transpose(1, 2)
129
+ else:
130
+ initial_states = None
131
+ x_ref = x.detach().clone()
132
+ weight_ref = weight.detach().clone()
133
+ bias_ref = bias.detach().clone() if bias is not None else None
134
+ initial_states_ref = initial_states.detach().clone(
135
+ ) if initial_states is not None else None
136
+ activation = None if not silu_activation else "silu"
137
+ out, final_states = causal_conv1d_fn(
138
+ x,
139
+ weight,
140
+ bias,
141
+ initial_states=initial_states,
142
+ return_final_states=return_final_states,
143
+ activation=activation)
144
+ out_ref, final_states_ref = causal_conv1d_ref(
145
+ x_ref,
146
+ weight_ref,
147
+ bias_ref,
148
+ initial_states=initial_states_ref,
149
+ return_final_states=return_final_states,
150
+ activation=activation)
151
+ if return_final_states:
152
+ assert final_states is not None and final_states_ref is not None
153
+ assert torch.allclose(final_states,
154
+ final_states_ref,
155
+ rtol=rtol,
156
+ atol=atol)
157
+
158
+ assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
159
+
160
+ if return_final_states:
161
+ out += F.sigmoid(final_states).sum(dim=-1, keepdim=True)
162
+ out_ref += F.sigmoid(final_states_ref).sum(dim=-1, keepdim=True)
163
+
164
+
165
+@pytest.mark.parametrize("itype", [torch.bfloat16])
166
+@pytest.mark.parametrize("silu_activation", [False, True])
167
+@pytest.mark.parametrize("has_bias", [False, True])
168
+@pytest.mark.parametrize("width", [2, 3, 4])
169
+@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
170
+@pytest.mark.parametrize("batch", [1, 2])
171
+def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
172
+ itype):
173
+ device = "cuda"
174
+ rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
175
+ if itype == torch.bfloat16:
176
+ rtol, atol = 1e-2, 5e-2
177
+ # set seed
178
+ torch.random.manual_seed(0)
179
+ batch = 2
180
+ x = torch.randn(batch, dim, device=device, dtype=itype)
181
+ conv_state = torch.randn(batch, dim, width, device=device, dtype=itype)
182
+ weight = torch.randn(dim,
183
+ width,
184
+ device=device,
185
+ dtype=itype,
186
+ requires_grad=True)
187
+ if has_bias:
188
+ bias = torch.randn(dim, device=device, dtype=itype, requires_grad=True)
189
+ else:
190
+ bias = None
191
+ conv_state_ref = conv_state.detach().clone()
192
+ activation = None if not silu_activation else "silu"
193
+ out = causal_conv1d_update(x,
194
+ conv_state,
195
+ weight,
196
+ bias,
197
+ activation=activation)
198
+ out_ref = causal_conv1d_update_ref(x,
199
+ conv_state_ref,
200
+ weight,
201
+ bias,
202
+ activation=activation)
203
+
204
+ assert torch.equal(conv_state, conv_state_ref)
205
+ assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
tests/kernels/test_mamba_ssm.py
new
+324
@@ -0,0 +1,324 @@
1
+import pytest
2
+import torch
3
+import torch.nn.functional as F
4
+from einops import rearrange, repeat
5
+
6
+from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
7
+ selective_scan_fn, selective_state_update)
8
+
9
+
10
+def selective_state_update_ref(state,
11
+ x,
12
+ dt,
13
+ A,
14
+ B,
15
+ C,
16
+ D=None,
17
+ z=None,
18
+ dt_bias=None,
19
+ dt_softplus=False):
20
+ """
21
+ Argument:
22
+ state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
23
+ x: (batch, dim) or (batch, nheads, dim)
24
+ dt: (batch, dim) or (batch, nheads, dim)
25
+ A: (dim, dstate) or (nheads, dim, dstate)
26
+ B: (batch, dstate) or (batch, ngroups, dstate)
27
+ C: (batch, dstate) or (batch, ngroups, dstate)
28
+ D: (dim,) or (nheads, dim)
29
+ z: (batch, dim) or (batch, nheads, dim)
30
+ dt_bias: (dim,) or (nheads, dim)
31
+ Return:
32
+ out: (batch, dim) or (batch, nheads, dim)
33
+ """
34
+ has_heads = state.dim() > 3
35
+ if state.dim() == 3:
36
+ state = state.unsqueeze(1)
37
+ if x.dim() == 2:
38
+ x = x.unsqueeze(1)
39
+ if dt.dim() == 2:
40
+ dt = dt.unsqueeze(1)
41
+ if A.dim() == 2:
42
+ A = A.unsqueeze(0)
43
+ if B.dim() == 2:
44
+ B = B.unsqueeze(1)
45
+ if C.dim() == 2:
46
+ C = C.unsqueeze(1)
47
+ if D is not None and D.dim() == 1:
48
+ D = D.unsqueeze(0)
49
+ if z is not None and z.dim() == 2:
50
+ z = z.unsqueeze(1)
51
+ if dt_bias is not None and dt_bias.dim() == 1:
52
+ dt_bias = dt_bias.unsqueeze(0)
53
+ batch, nheads, dim, dstate = state.shape
54
+ assert x.shape == (batch, nheads, dim)
55
+ assert dt.shape == x.shape
56
+ assert A.shape == (nheads, dim, dstate)
57
+ ngroups = B.shape[1]
58
+ assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
59
+ assert B.shape == (batch, ngroups, dstate)
60
+ assert C.shape == B.shape
61
+ if D is not None:
62
+ assert D.shape == (nheads, dim)
63
+ if z is not None:
64
+ assert z.shape == x.shape
65
+ if dt_bias is not None:
66
+ assert dt_bias.shape == (nheads, dim)
67
+ dt = dt + dt_bias
68
+ dt = F.softplus(dt) if dt_softplus else dt
69
+ dA = torch.exp(rearrange(dt, "b h d -> b h d 1") *
70
+ A) # (batch, nheads, dim, dstate)
71
+ B = repeat(B, "b g n -> b (g h) n",
72
+ h=nheads // ngroups) # (batch, nheads, dstate)
73
+ C = repeat(C, "b g n -> b (g h) n",
74
+ h=nheads // ngroups) # (batch, nheads, dstate)
75
+ dB = rearrange(dt, "b h d -> b h d 1") * rearrange(
76
+ B, "b h n -> b h 1 n") # (batch, nheads, dim, dstate)
77
+ state.copy_(state * dA +
78
+ dB * rearrange(x, "b h d -> b h d 1")) # (batch, dim, dstate
79
+ out = torch.einsum("bhdn,bhn->bhd", state.to(C.dtype), C)
80
+ if D is not None:
81
+ out += (x * D).to(out.dtype)
82
+ out = (out if z is None else out * F.silu(z)).to(x.dtype)
83
+ if not has_heads:
84
+ out = out.squeeze(1)
85
+ return out
86
+
87
+
88
+def selective_scan_ref(u,
89
+ delta,
90
+ A,
91
+ B,
92
+ C,
93
+ D=None,
94
+ z=None,
95
+ delta_bias=None,
96
+ delta_softplus=False,
97
+ return_last_state=False,
98
+ position_indices=None,
99
+ prev_state=None):
100
+ """
101
+ u: r(B D L)
102
+ delta: r(B D L)
103
+ A: c(D N) or r(D N)
104
+ B: c(D N) or r(B N L) or r(B N 2L) or r(B G N L) or (B G N L)
105
+ C: c(D N) or r(B N L) or r(B N 2L) or r(B G N L) or (B G N L)
106
+ D: r(D)
107
+ z: r(B D L)
108
+ delta_bias: r(D), fp32
109
+ prev_state: r(B D N), fp32
110
+
111
+ out: r(B D L)
112
+ last_state (optional): r(B D dstate) or c(B D dstate)
113
+ """
114
+ dtype_in = u.dtype
115
+ u = u.float()
116
+ delta = delta.float()
117
+ if delta_bias is not None:
118
+ delta = delta + delta_bias[..., None].float()
119
+ if delta_softplus:
120
+ delta = F.softplus(delta)
121
+ batch, dim, dstate = u.shape[0], A.shape[0], A.shape[1]
122
+ is_variable_B = B.dim() >= 3
123
+ is_variable_C = C.dim() >= 3
124
+ B = B.float()
125
+ C = C.float()
126
+ x = A.new_zeros((batch, dim, dstate)) if prev_state is None else prev_state
127
+ ys = []
128
+ deltaA = torch.exp(torch.einsum('bdl,dn->bdln', delta, A))
129
+ if not is_variable_B:
130
+ deltaB_u = torch.einsum('bdl,dn,bdl->bdln', delta, B, u)
131
+ else:
132
+ if B.dim() == 3:
133
+ deltaB_u = torch.einsum('bdl,bnl,bdl->bdln', delta, B, u)
134
+ else:
135
+ B = repeat(B, "B G N L -> B (G H) N L", H=dim // B.shape[1])
136
+ deltaB_u = torch.einsum('bdl,bdnl,bdl->bdln', delta, B, u)
137
+ if is_variable_C and C.dim() == 4:
138
+ C = repeat(C, "B G N L -> B (G H) N L", H=dim // C.shape[1])
139
+ last_state = None
140
+ for i in range(u.shape[2]):
141
+ if position_indices is not None and position_indices[0, i] == 0:
142
+ x = deltaB_u[:, :, i]
143
+ else:
144
+ x = deltaA[:, :, i] * x + deltaB_u[:, :, i]
145
+ if not is_variable_C:
146
+ y = torch.einsum('bdn,dn->bd', x, C)
147
+ else:
148
+ if C.dim() == 3:
149
+ y = torch.einsum('bdn,bn->bd', x, C[:, :, i])
150
+ else:
151
+ y = torch.einsum('bdn,bdn->bd', x, C[:, :, :, i])
152
+ if i == u.shape[2] - 1:
153
+ last_state = x
154
+ ys.append(y)
155
+ y = torch.stack(ys, dim=2) # (batch dim L)
156
+ out = y if D is None else y + u * rearrange(D, "d -> d 1")
157
+ if z is not None:
158
+ out = out * F.silu(z)
159
+ out = out.to(dtype=dtype_in)
160
+ return out if not return_last_state else (out, last_state)
161
+
162
+
163
+@pytest.mark.parametrize('wtype', [torch.float32])
164
+@pytest.mark.parametrize('itype', [torch.float32])
165
+@pytest.mark.parametrize('seqlen', [128, 256, 512, 1024, 2048, 4096])
166
+@pytest.mark.parametrize("return_last_state", [True])
167
+@pytest.mark.parametrize('has_delta_bias', [True])
168
+@pytest.mark.parametrize('delta_softplus', [True])
169
+@pytest.mark.parametrize('has_z', [True])
170
+@pytest.mark.parametrize('has_D', [True])
171
+@pytest.mark.parametrize("varBC_groups", [1, 2])
172
+@pytest.mark.parametrize("is_variable_C", [True])
173
+@pytest.mark.parametrize("is_variable_B", [True])
174
+@pytest.mark.parametrize("scan_chunks", [1, 2, 3])
175
+def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
176
+ has_z, has_delta_bias, delta_softplus,
177
+ return_last_state, seqlen, itype, wtype, scan_chunks):
178
+ if varBC_groups > 1 and (not is_variable_B or not is_variable_C):
179
+ pytest.skip() # This config is not applicable
180
+ device = 'cuda'
181
+ rtol, atol = (6e-4, 2e-3) if itype == torch.float32 else (3e-3, 5e-3)
182
+ if itype == torch.bfloat16:
183
+ rtol, atol = 3e-2, 5e-2
184
+ rtolw, atolw = (1e-3, 1e-3)
185
+ if has_z: # If we have z, the errors on the weights seem higher
186
+ rtolw = max(rtolw, rtol)
187
+ atolw = max(atolw, atol)
188
+ # set seed
189
+ torch.random.manual_seed(0)
190
+ batch_size = 2
191
+ dim = 4
192
+ dstate = 8
193
+ A = (-0.5 * torch.rand(dim, dstate, device=device, dtype=wtype))
194
+ if not is_variable_B:
195
+ B_shape = [dim, dstate]
196
+ elif varBC_groups == 1:
197
+ B_shape = [batch_size, dstate, seqlen]
198
+ else:
199
+ B_shape = [batch_size, varBC_groups, dstate, seqlen]
200
+ B = torch.randn(B_shape,
201
+ device=device,
202
+ dtype=wtype if not is_variable_B else itype)
203
+ if not is_variable_C:
204
+ C_shape = [dim, dstate]
205
+ elif varBC_groups == 1:
206
+ C_shape = [batch_size, dstate, seqlen]
207
+ else:
208
+ C_shape = [batch_size, varBC_groups, dstate, seqlen]
209
+ C = torch.randn(C_shape,
210
+ device=device,
211
+ dtype=wtype if not is_variable_C else itype)
212
+ D = torch.randn(dim, device=device, dtype=torch.float32) if has_D else None
213
+ z = torch.randn(batch_size, dim, seqlen, device=device,
214
+ dtype=itype) if has_z else None
215
+ delta_bias = (0.5 * torch.rand(dim, device=device, dtype=torch.float32)
216
+ ) if has_delta_bias else None
217
+ u = torch.randn(batch_size, dim, seqlen, device=device, dtype=itype)
218
+ delta = (0.5 *
219
+ torch.rand(batch_size, dim, seqlen, device=device, dtype=itype))
220
+ state = None
221
+ state_ref = None
222
+ out = None
223
+ out_ref = None
224
+ outs = []
225
+ for c in range(scan_chunks):
226
+ chunked_prompt_len = seqlen // scan_chunks
227
+ chunk_start = chunked_prompt_len * c
228
+ chunk_end = chunked_prompt_len * (c + 1)
229
+ if c == scan_chunks - 1:
230
+ chunk_end = seqlen
231
+ _B = B
232
+ if is_variable_B:
233
+ _B = B[..., chunk_start:chunk_end]
234
+ _C = C
235
+ if is_variable_B:
236
+ _C = C[..., chunk_start:chunk_end]
237
+ _z = z
238
+ if has_z:
239
+ assert z is not None
240
+ _z = z[..., chunk_start:chunk_end]
241
+ out, *rest = selective_scan_fn(u[..., chunk_start:chunk_end],
242
+ delta[..., chunk_start:chunk_end],
243
+ A,
244
+ _B,
245
+ _C,
246
+ D,
247
+ z=_z,
248
+ delta_bias=delta_bias,
249
+ delta_softplus=delta_softplus,
250
+ return_last_state=return_last_state,
251
+ prev_state=state if c > 0 else None)
252
+ outs.append(out)
253
+ if return_last_state:
254
+ state = rest[0]
255
+ if len(outs) > 1:
256
+ out = torch.cat(outs, dim=-1)
257
+ out_ref, *rest = selective_scan_ref(u,
258
+ delta,
259
+ A,
260
+ B,
261
+ C,
262
+ D,
263
+ z=z,
264
+ delta_bias=delta_bias,
265
+ delta_softplus=delta_softplus,
266
+ return_last_state=return_last_state)
267
+ if return_last_state:
268
+ state_ref = rest[0]
269
+
270
+ assert out is not None and out_ref is not None
271
+ assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
272
+ if return_last_state:
273
+ assert state is not None and state_ref is not None
274
+ assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
275
+
276
+
277
+@pytest.mark.parametrize("itype",
278
+ [torch.float32, torch.float16, torch.bfloat16])
279
+@pytest.mark.parametrize("has_z", [False, True])
280
+@pytest.mark.parametrize("dstate", [16, 32, 64])
281
+@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
282
+def test_selective_state_update(dim, dstate, has_z, itype):
283
+ device = "cuda"
284
+ rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
285
+ if itype == torch.bfloat16:
286
+ rtol, atol = 1e-2, 5e-2
287
+ if torch.version.hip:
288
+ atol *= 2
289
+ # set seed
290
+ torch.random.manual_seed(0)
291
+ batch_size = 1
292
+ state = torch.randn(batch_size, dim, dstate, dtype=itype, device=device)
293
+ x = torch.randn(batch_size, dim, device=device, dtype=itype)
294
+ dt = torch.randn(batch_size, dim, device=device, dtype=itype)
295
+ dt_bias = torch.rand(dim, device=device) - 4.0
296
+ A = -torch.rand(dim, dstate, device=device) - 1.0
297
+ B = torch.randn(batch_size, dstate, device=device)
298
+ C = torch.randn(batch_size, dstate, device=device)
299
+ D = torch.randn(dim, device=device)
300
+ z = torch.randn_like(x) if has_z else None
301
+ state_ref = state.detach().clone()
302
+ out = selective_state_update(state,
303
+ x,
304
+ dt,
305
+ A,
306
+ B,
307
+ C,
308
+ D=D,
309
+ z=z,
310
+ dt_bias=dt_bias,
311
+ dt_softplus=True)
312
+ out_ref = selective_state_update_ref(state_ref,
313
+ x,
314
+ dt,
315
+ A,
316
+ B,
317
+ C,
318
+ D=D,
319
+ z=z,
320
+ dt_bias=dt_bias,
321
+ dt_softplus=True)
322
+
323
+ assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
324
+ assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
vllm/_custom_ops.py
+30
@@ -500,6 +500,36 @@ def ggml_mul_mat_a8(
500
return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
501
502
503
+# mamba
504
+def causal_conv1d_fwd(x: torch.Tensor, weight: torch.Tensor,
505
+ bias_: Optional[torch.Tensor],
506
+ seq_idx_: Optional[torch.Tensor],
507
+ initial_states_: Optional[torch.Tensor],
508
+ final_states_out_: Optional[torch.Tensor],
509
+ silu_activation: bool) -> torch.Tensor:
510
+ return torch.ops._C.causal_conv1d_fwd(x, weight, bias_, seq_idx_,
511
+ initial_states_, final_states_out_,
512
+ silu_activation)
513
+
514
+
515
+def causal_conv1d_update(x: torch.Tensor, conv_state: torch.Tensor,
516
+ weight: torch.Tensor, bias_: Optional[torch.Tensor],
517
+ silu_activation: bool) -> torch.Tensor:
518
+ return torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
519
+ silu_activation)
520
+
521
+
522
+def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
523
+ B: torch.Tensor, C: torch.Tensor,
524
+ D_: Optional[torch.Tensor], z_: Optional[torch.Tensor],
525
+ delta_bias_: Optional[torch.Tensor],
526
+ delta_softplus: bool, index_: Optional[torch.Tensor],
527
+ x: Optional[torch.Tensor]) -> List[torch.Tensor]:
528
+ return torch.ops._C.selective_scan_fwd(u, delta, A, B, C, D_, z_,
529
+ delta_bias_, delta_softplus, index_,
530
+ x)
531
+
532
+
533
# moe
534
def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
535
block_size: int, sorted_token_ids: torch.Tensor,
vllm/model_executor/layers/mamba/__init__.py
vllm/model_executor/layers/mamba/ops/__init__.py
vllm/model_executor/layers/mamba/ops/causal_conv1d.py
new
+86
@@ -0,0 +1,86 @@
1
+# Copyright (c) 2024, Tri Dao.
2
+
3
+from typing import Optional
4
+
5
+import torch
6
+
7
+from vllm import _custom_ops as ops
8
+
9
+
10
+def causal_conv1d_fn(
11
+ x: torch.Tensor,
12
+ weight: torch.Tensor,
13
+ bias: Optional[torch.Tensor] = None,
14
+ seq_idx: Optional[torch.Tensor] = None,
15
+ initial_states: Optional[torch.Tensor] = None,
16
+ return_final_states: bool = False,
17
+ final_states_out=None,
18
+ activation: str = "silu",
19
+):
20
+ """
21
+ x: (batch, dim, seqlen)
22
+ weight: (dim, width)
23
+ bias: (dim,)
24
+ seq_idx: (batch, seqlen)
25
+ initial_states: (batch, dim, width - 1)
26
+ final_states_out: (batch, dim, width - 1), to be written to
27
+ activation: either None or "silu" or "swish"
28
+
29
+ out: (batch, dim, seqlen)
30
+ """
31
+ if activation not in [None, "silu", "swish"]:
32
+ raise NotImplementedError("activation must be None, silu, or swish")
33
+ if x.stride(2) != 1 and x.stride(1) != 1:
34
+ x = x.contiguous()
35
+ bias = bias.contiguous() if bias is not None else None
36
+ if seq_idx is not None:
37
+ assert (initial_states is
38
+ None), "initial_states must be None if seq_idx is not None"
39
+ assert (not return_final_states
40
+ ), "If seq_idx is not None, we don't return final_states_out"
41
+ seq_idx = seq_idx.contiguous() if seq_idx is not None else None
42
+ if initial_states is not None and (initial_states.stride(2) != 1
43
+ and initial_states.stride(1) != 1):
44
+ initial_states = initial_states.contiguous()
45
+ if return_final_states:
46
+ assert (
47
+ x.stride(1) == 1
48
+ ), "Only channel-last layout support returning final_states_out"
49
+ if final_states_out is not None:
50
+ assert (final_states_out.stride(2) == 1
51
+ or final_states_out.stride(1) == 1)
52
+ else:
53
+ batch, dim, seqlen = x.shape
54
+ width = weight.shape[1]
55
+ final_states_out = torch.empty(batch,
56
+ width - 1,
57
+ dim,
58
+ device=x.device,
59
+ dtype=x.dtype).transpose(1, 2)
60
+ else:
61
+ final_states_out = None
62
+
63
+ out = ops.causal_conv1d_fwd(x, weight, bias, seq_idx, initial_states,
64
+ final_states_out, activation
65
+ in ["silu", "swish"])
66
+ return (out, None) if not return_final_states else (out, final_states_out)
67
+
68
+
69
+def causal_conv1d_update(x: torch.Tensor,
70
+ conv_state: torch.Tensor,
71
+ weight: torch.Tensor,
72
+ bias: Optional[torch.Tensor] = None,
73
+ activation: Optional[str] = None):
74
+ """
75
+ x: (batch, dim)
76
+ conv_state: (batch, dim, width)
77
+ weight: (dim, width)
78
+ bias: (dim,)
79
+
80
+ out: (batch, dim)
81
+ """
82
+ if activation not in [None, "silu", "swish"]:
83
+ raise NotImplementedError("activation must be None, silu, or swish")
84
+ activation_bool = activation in ["silu", "swish"]
85
+ return ops.causal_conv1d_update(x, conv_state, weight, bias,
86
+ activation_bool)
vllm/model_executor/layers/mamba/ops/mamba_ssm.py
new
+346
@@ -0,0 +1,346 @@
1
+# Copyright (c) 2024, Tri Dao, Albert Gu.
2
+
3
+import torch
4
+import triton
5
+import triton.language as tl
6
+from packaging import version
7
+
8
+from vllm import _custom_ops as ops
9
+
10
+TRITON3 = version.parse(triton.__version__) >= version.parse("3.0.0")
11
+
12
+if TRITON3:
13
+
14
+ @triton.jit
15
+ def softplus(dt):
16
+ dt = tl.where(dt <= 20.0, tl.math.log(tl.math.exp(dt) + 1), dt)
17
+ return dt
18
+else:
19
+
20
+ @triton.jit
21
+ def softplus(dt):
22
+ dt = tl.where(dt <= 20.0, tl.math.log1p(tl.exp(dt)), dt)
23
+ return dt
24
+
25
+
26
+@triton.heuristics(
27
+ {"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None})
28
+@triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None})
29
+@triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None})
30
+@triton.heuristics(
31
+ {"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])})
32
+@triton.jit
33
+def _selective_scan_update_kernel(
34
+ # Pointers to matrices
35
+ state_ptr,
36
+ x_ptr,
37
+ dt_ptr,
38
+ dt_bias_ptr,
39
+ A_ptr,
40
+ B_ptr,
41
+ C_ptr,
42
+ D_ptr,
43
+ z_ptr,
44
+ out_ptr,
45
+ # Matrix dimensions
46
+ batch,
47
+ nheads,
48
+ dim,
49
+ dstate,
50
+ nheads_ngroups_ratio,
51
+ # Strides
52
+ stride_state_batch,
53
+ stride_state_head,
54
+ stride_state_dim,
55
+ stride_state_dstate,
56
+ stride_x_batch,
57
+ stride_x_head,
58
+ stride_x_dim,
59
+ stride_dt_batch,
60
+ stride_dt_head,
61
+ stride_dt_dim,
62
+ stride_dt_bias_head,
63
+ stride_dt_bias_dim,
64
+ stride_A_head,
65
+ stride_A_dim,
66
+ stride_A_dstate,
67
+ stride_B_batch,
68
+ stride_B_group,
69
+ stride_B_dstate,
70
+ stride_C_batch,
71
+ stride_C_group,
72
+ stride_C_dstate,
73
+ stride_D_head,
74
+ stride_D_dim,
75
+ stride_z_batch,
76
+ stride_z_head,
77
+ stride_z_dim,
78
+ stride_out_batch,
79
+ stride_out_head,
80
+ stride_out_dim,
81
+ # Meta-parameters
82
+ DT_SOFTPLUS: tl.constexpr,
83
+ TIE_HDIM: tl.constexpr,
84
+ BLOCK_SIZE_M: tl.constexpr,
85
+ HAS_DT_BIAS: tl.constexpr,
86
+ HAS_D: tl.constexpr,
87
+ HAS_Z: tl.constexpr,
88
+ BLOCK_SIZE_DSTATE: tl.constexpr,
89
+):
90
+ pid_m = tl.program_id(axis=0)
91
+ pid_b = tl.program_id(axis=1)
92
+ pid_h = tl.program_id(axis=2)
93
+ state_ptr += pid_b * stride_state_batch + pid_h * stride_state_head
94
+ x_ptr += pid_b * stride_x_batch + pid_h * stride_x_head
95
+ dt_ptr += pid_b * stride_dt_batch + pid_h * stride_dt_head
96
+ if HAS_DT_BIAS:
97
+ dt_bias_ptr += pid_h * stride_dt_bias_head
98
+ A_ptr += pid_h * stride_A_head
99
+ B_ptr += pid_b * stride_B_batch + (pid_h //
100
+ nheads_ngroups_ratio) * stride_B_group
101
+ C_ptr += pid_b * stride_C_batch + (pid_h //
102
+ nheads_ngroups_ratio) * stride_C_group
103
+ if HAS_Z:
104
+ z_ptr += pid_b * stride_z_batch + pid_h * stride_z_head
105
+ out_ptr += pid_b * stride_out_batch + pid_h * stride_out_head
106
+
107
+ offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
108
+ offs_n = tl.arange(0, BLOCK_SIZE_DSTATE)
109
+ state_ptrs = state_ptr + (offs_m[:, None] * stride_state_dim +
110
+ offs_n[None, :] * stride_state_dstate)
111
+ x_ptrs = x_ptr + offs_m * stride_x_dim
112
+ dt_ptrs = dt_ptr + offs_m * stride_dt_dim
113
+ if HAS_DT_BIAS:
114
+ dt_bias_ptrs = dt_bias_ptr + offs_m * stride_dt_bias_dim
115
+ if HAS_D:
116
+ D_ptr += pid_h * stride_D_head
117
+ A_ptrs = A_ptr + (offs_m[:, None] * stride_A_dim +
118
+ offs_n[None, :] * stride_A_dstate)
119
+ B_ptrs = B_ptr + offs_n * stride_B_dstate
120
+ C_ptrs = C_ptr + offs_n * stride_C_dstate
121
+ if HAS_D:
122
+ D_ptrs = D_ptr + offs_m * stride_D_dim
123
+ if HAS_Z:
124
+ z_ptrs = z_ptr + offs_m * stride_z_dim
125
+ out_ptrs = out_ptr + offs_m * stride_out_dim
126
+
127
+ state = tl.load(state_ptrs,
128
+ mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate),
129
+ other=0.0)
130
+ x = tl.load(x_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
131
+ if not TIE_HDIM:
132
+ dt = tl.load(dt_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
133
+ if HAS_DT_BIAS:
134
+ dt += tl.load(dt_bias_ptrs, mask=offs_m < dim,
135
+ other=0.0).to(tl.float32)
136
+ if DT_SOFTPLUS:
137
+ dt = softplus(dt)
138
+ A = tl.load(A_ptrs,
139
+ mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate),
140
+ other=0.0).to(tl.float32)
141
+ dA = tl.exp(A * dt[:, None])
142
+ else:
143
+ dt = tl.load(dt_ptr).to(tl.float32)
144
+ if HAS_DT_BIAS:
145
+ dt += tl.load(dt_bias_ptr).to(tl.float32)
146
+ if DT_SOFTPLUS:
147
+ dt = softplus(dt)
148
+ A = tl.load(A_ptr).to(tl.float32)
149
+ dA = tl.exp(A * dt) # scalar, not a matrix
150
+
151
+ B = tl.load(B_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
152
+ C = tl.load(C_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
153
+ if HAS_D:
154
+ D = tl.load(D_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
155
+ if HAS_Z:
156
+ z = tl.load(z_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
157
+
158
+ dB = B[None, :] * dt[:, None] if not TIE_HDIM else B * dt
159
+ state = state * dA + dB * x[:, None]
160
+ tl.store(state_ptrs,
161
+ state,
162
+ mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate))
163
+ out = tl.sum(state * C[None, :], axis=1)
164
+ if HAS_D:
165
+ out += x * D
166
+ if HAS_Z:
167
+ out *= z * tl.sigmoid(z)
168
+ tl.store(out_ptrs, out, mask=offs_m < dim)
169
+
170
+
171
+def selective_state_update(state,
172
+ x,
173
+ dt,
174
+ A,
175
+ B,
176
+ C,
177
+ D=None,
178
+ z=None,
179
+ dt_bias=None,
180
+ dt_softplus=False):
181
+ """
182
+ Argument:
183
+ state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
184
+ x: (batch, dim) or (batch, nheads, dim)
185
+ dt: (batch, dim) or (batch, nheads, dim)
186
+ A: (dim, dstate) or (nheads, dim, dstate)
187
+ B: (batch, dstate) or (batch, ngroups, dstate)
188
+ C: (batch, dstate) or (batch, ngroups, dstate)
189
+ D: (dim,) or (nheads, dim)
190
+ z: (batch, dim) or (batch, nheads, dim)
191
+ dt_bias: (dim,) or (nheads, dim)
192
+ Return:
193
+ out: (batch, dim) or (batch, nheads, dim)
194
+ """
195
+ has_heads = state.dim() > 3
196
+ if state.dim() == 3:
197
+ state = state.unsqueeze(1)
198
+ if x.dim() == 2:
199
+ x = x.unsqueeze(1)
200
+ if dt.dim() == 2:
201
+ dt = dt.unsqueeze(1)
202
+ if A.dim() == 2:
203
+ A = A.unsqueeze(0)
204
+ if B.dim() == 2:
205
+ B = B.unsqueeze(1)
206
+ if C.dim() == 2:
207
+ C = C.unsqueeze(1)
208
+ if D is not None and D.dim() == 1:
209
+ D = D.unsqueeze(0)
210
+ if z is not None and z.dim() == 2:
211
+ z = z.unsqueeze(1)
212
+ if dt_bias is not None and dt_bias.dim() == 1:
213
+ dt_bias = dt_bias.unsqueeze(0)
214
+ batch, nheads, dim, dstate = state.shape
215
+ assert x.shape == (batch, nheads, dim)
216
+ assert dt.shape == x.shape
217
+ assert A.shape == (nheads, dim, dstate)
218
+ ngroups = B.shape[1]
219
+ assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
220
+ assert B.shape == (batch, ngroups, dstate)
221
+ assert C.shape == B.shape
222
+ if D is not None:
223
+ assert D.shape == (nheads, dim)
224
+ if z is not None:
225
+ assert z.shape == x.shape
226
+ if dt_bias is not None:
227
+ assert dt_bias.shape == (nheads, dim)
228
+ out = torch.empty_like(x)
229
+ grid = lambda META: (triton.cdiv(dim, META['BLOCK_SIZE_M']), batch, nheads)
230
+ z_strides = ((z.stride(0), z.stride(1), z.stride(2)) if z is not None else
231
+ (0, 0, 0))
232
+ # We don't want autotune since it will overwrite the state
233
+ # We instead tune by hand.
234
+ BLOCK_SIZE_M, num_warps = ((32, 4) if dstate <= 16 else
235
+ ((16, 4) if dstate <= 32 else
236
+ ((8, 4) if dstate <= 64 else
237
+ ((4, 4) if dstate <= 128 else ((4, 8))))))
238
+ tie_hdim = A.stride(-1) == 0 and A.stride(-2) == 0 and dt.stride(
239
+ -1) == 0 and dt_bias.stride(-1) == 0
240
+ with torch.cuda.device(x.device.index):
241
+ _selective_scan_update_kernel[grid](
242
+ state,
243
+ x,
244
+ dt,
245
+ dt_bias,
246
+ A,
247
+ B,
248
+ C,
249
+ D,
250
+ z,
251
+ out,
252
+ batch,
253
+ nheads,
254
+ dim,
255
+ dstate,
256
+ nheads // ngroups,
257
+ state.stride(0),
258
+ state.stride(1),
259
+ state.stride(2),
260
+ state.stride(3),
261
+ x.stride(0),
262
+ x.stride(1),
263
+ x.stride(2),
264
+ dt.stride(0),
265
+ dt.stride(1),
266
+ dt.stride(2),
267
+ *(dt_bias.stride(0),
268
+ dt_bias.stride(1)) if dt_bias is not None else 0,
269
+ A.stride(0),
270
+ A.stride(1),
271
+ A.stride(2),
272
+ B.stride(0),
273
+ B.stride(1),
274
+ B.stride(2),
275
+ C.stride(0),
276
+ C.stride(1),
277
+ C.stride(2),
278
+ *(D.stride(0), D.stride(1)) if D is not None else 0,
279
+ z_strides[0],
280
+ z_strides[1],
281
+ z_strides[2],
282
+ out.stride(0),
283
+ out.stride(1),
284
+ out.stride(2),
285
+ dt_softplus,
286
+ tie_hdim,
287
+ BLOCK_SIZE_M,
288
+ num_warps=num_warps,
289
+ )
290
+ if not has_heads:
291
+ out = out.squeeze(1)
292
+ return out
293
+
294
+
295
+def selective_scan_fn(u,
296
+ delta,
297
+ A,
298
+ B,
299
+ C,
300
+ D=None,
301
+ z=None,
302
+ delta_bias=None,
303
+ delta_softplus=False,
304
+ return_last_state=False,
305
+ position_indices=None,
306
+ prev_state=None):
307
+ """if return_last_state is True, returns (out, last_state)
308
+ last_state has shape (batch, dim, dstate).
309
+ """
310
+ if u.stride(-1) != 1:
311
+ u = u.contiguous()
312
+ if delta.stride(-1) != 1:
313
+ delta = delta.contiguous()
314
+ if D is not None:
315
+ D = D.contiguous()
316
+ if B.stride(-1) != 1:
317
+ B = B.contiguous()
318
+ if C.stride(-1) != 1:
319
+ C = C.contiguous()
320
+ if z is not None and z.stride(-1) != 1:
321
+ z = z.contiguous()
322
+ if B.dim() == 3:
323
+ B = B.unsqueeze(1)
324
+ if C.dim() == 3:
325
+ C = C.unsqueeze(1)
326
+ n_chunks = int((u.shape[-1] + 2048 - 1) / 2048)
327
+ x = torch.zeros((
328
+ u.shape[0],
329
+ u.shape[1],
330
+ n_chunks,
331
+ int(A.shape[1] * 2),
332
+ ),
333
+ device=u.device,
334
+ dtype=torch.float32,
335
+ requires_grad=False)
336
+ x[:, :, 0, 0::2] = 1
337
+ if prev_state is not None:
338
+ x[:, :, 0, 1::2].copy_(prev_state)
339
+ out, x, *rest = ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias,
340
+ delta_softplus, position_indices, x)
341
+ last_state = x[:, :, -1, 1::2] # (batch, dim, dstate)
342
+ if z is None:
343
+ return out if not return_last_state else (out, last_state)
344
+ else:
345
+ out_z = rest[0]
346
+ return out_z if not return_last_state else (out_z, last_state)
vllm/model_executor/models/jamba.py
+5
-4
@@ -4,9 +4,6 @@ from dataclasses import dataclass
4
from typing import Dict, Iterable, List, Optional, Tuple
5
6
import torch
7
-from causal_conv1d import causal_conv1d_fn, causal_conv1d_update
8
-from mamba_ssm.ops.selective_scan_interface import selective_scan_fn
9
-from mamba_ssm.ops.triton.selective_state_update import selective_state_update
7
from torch import nn
8
from torch.nn.parameter import Parameter
9
from transformers import JambaConfig
@@ -24,6 +21,10 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
21
ReplicatedLinear,
22
RowParallelLinear)
23
from vllm.model_executor.layers.logits_processor import LogitsProcessor
24
+from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
25
+ causal_conv1d_fn, causal_conv1d_update)
26
+from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
27
+ selective_scan_fn, selective_state_update)
28
from vllm.model_executor.layers.quantization.base_config import (
29
QuantizationConfig)
30
from vllm.model_executor.layers.sampler import Sampler
@@ -161,7 +162,7 @@ class JambaMambaMixer(nn.Module):
162
(self.conv_kernel_size - hidden_states.shape[-1], 0))
163
cache_params.conv_state.copy_(conv_states)
164
164
- hidden_states = causal_conv1d_fn(
165
+ hidden_states, _ = causal_conv1d_fn(
166
hidden_states,
167
conv_weights,
168
self.conv1d.bias,