@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fdd9daafa3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Kernel/Model] Migrate mamba_ssm and causal_conv1d kernels to vLLM (#7651)

Mor Zusman committed Aug 29, 2024 at 01:06 UTC fdd9daafa3b31746ec8ec7c0d67ebc7efeb13f8f
20 files changed +2815 -31
CMakeLists.txt
+2
@@ -203,6 +203,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
203 FetchContent_MakeAvailable(cutlass)
204
205 list(APPEND VLLM_EXT_SRC
206 + "csrc/mamba/mamba_ssm/selective_scan_fwd.cu"
207 + "csrc/mamba/causal_conv1d/causal_conv1d.cu"
208 "csrc/quantization/aqlm/gemm_kernels.cu"
209 "csrc/quantization/awq/gemm_kernels.cu"
210 "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
Dockerfile
-23
@@ -42,9 +42,6 @@ COPY requirements-cuda.txt requirements-cuda.txt
42 RUN --mount=type=cache,target=/root/.cache/pip \
43 python3 -m pip install -r requirements-cuda.txt
44
45 -COPY requirements-mamba.txt requirements-mamba.txt
46 -RUN python3 -m pip install packaging
47 -RUN python3 -m pip install -r requirements-mamba.txt
45
46 # cuda arch list used by torch
47 # can be useful for both `dev` and `test`
@@ -127,22 +124,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
124 python3 -m pip install -r requirements-dev.txt
125
126 #################### DEV IMAGE ####################
130 -#################### MAMBA Build IMAGE ####################
131 -FROM dev as mamba-builder
132 -# max jobs used for build
133 -ARG max_jobs=2
134 -ENV MAX_JOBS=${max_jobs}
135 -
136 -WORKDIR /usr/src/mamba
137 -
138 -COPY requirements-mamba.txt requirements-mamba.txt
139 -
140 -# Download the wheel or build it if a pre-compiled release doesn't exist
141 -RUN pip --verbose wheel -r requirements-mamba.txt \
142 - --no-build-isolation --no-deps --no-cache-dir
143 -
144 -#################### MAMBA Build IMAGE ####################
145 -
127 #################### vLLM installation IMAGE ####################
128 # image with vLLM installed
129 FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu20.04 AS vllm-base
@@ -179,10 +160,6 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
160 --mount=type=cache,target=/root/.cache/pip \
161 python3 -m pip install dist/*.whl --verbose
162
182 -RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamba \
183 - --mount=type=cache,target=/root/.cache/pip \
184 - python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
185 -
163 RUN --mount=type=cache,target=/root/.cache/pip \
164 . /etc/environment && \
165 python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.4/flashinfer-0.1.4+cu121torch2.4-cp${PYTHON_VERSION_STR}-cp${PYTHON_VERSION_STR}-linux_x86_64.whl
csrc/mamba/causal_conv1d/causal_conv1d.cu new
+700
@@ -0,0 +1,700 @@
1 +// clang-format off
2 +// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_fwd.cu
3 +// and https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_update.cu
4 +#include <torch/all.h>
5 +#include <ATen/cuda/CUDAContext.h>
6 +#include <c10/cuda/CUDAGuard.h>
7 +
8 +#include "causal_conv1d.h"
9 +#include <c10/util/BFloat16.h>
10 +#include <c10/util/Half.h>
11 +#include <c10/cuda/CUDAException.h> // For C10_CUDA_CHECK and C10_CUDA_KERNEL_LAUNCH_CHECK
12 +
13 +#include <cub/block/block_load.cuh>
14 +#include <cub/block/block_store.cuh>
15 +
16 +#include "static_switch.h"
17 +
18 +
19 +
20 +#define CHECK_SHAPE(x, ...) TORCH_CHECK(x.sizes() == torch::IntArrayRef({__VA_ARGS__}), #x " must have shape (" #__VA_ARGS__ ")")
21 +
22 +#define DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(ITYPE, NAME, ...) \
23 + if (ITYPE == at::ScalarType::Half) { \
24 + using input_t = at::Half; \
25 + using weight_t = at::Half; \
26 + __VA_ARGS__(); \
27 + } else if (ITYPE == at::ScalarType::BFloat16) { \
28 + using input_t = at::BFloat16; \
29 + using weight_t = at::BFloat16; \
30 + __VA_ARGS__(); \
31 + } else if (ITYPE == at::ScalarType::Float) { \
32 + using input_t = float; \
33 + using weight_t = float; \
34 + __VA_ARGS__(); \
35 + } else { \
36 + AT_ERROR(#NAME, " not implemented for input type '", toString(ITYPE), "'"); \
37 + }
38 +
39 +
40 +template<typename input_t, typename weight_t>
41 +void causal_conv1d_fwd_cuda(ConvParamsBase &params, cudaStream_t stream);
42 +template <typename input_t, typename weight_t>
43 +void causal_conv1d_channellast_fwd_cuda(ConvParamsBase &params, cudaStream_t stream);
44 +
45 +template<typename input_t, typename weight_t>
46 +void causal_conv1d_update_cuda(ConvParamsBase &params, cudaStream_t stream);
47 +
48 +void set_conv_params_fwd(ConvParamsBase &params,
49 + // sizes
50 + const size_t batch,
51 + const size_t dim,
52 + const size_t seqlen,
53 + const size_t width,
54 + // device pointers
55 + const at::Tensor x,
56 + const at::Tensor weight,
57 + const at::Tensor out,
58 + void* bias_ptr,
59 + bool silu_activation) {
60 +
61 + // Reset the parameters
62 + memset(&params, 0, sizeof(params));
63 +
64 + params.batch = batch;
65 + params.dim = dim;
66 + params.seqlen = seqlen;
67 + params.width = width;
68 +
69 + params.silu_activation = silu_activation;
70 +
71 + // Set the pointers and strides.
72 + params.x_ptr = x.data_ptr();
73 + params.weight_ptr = weight.data_ptr();
74 + params.bias_ptr = bias_ptr;
75 + params.out_ptr = out.data_ptr();
76 + // All stride are in elements, not bytes.
77 + params.x_batch_stride = x.stride(0);
78 + params.x_c_stride = x.stride(1);
79 + params.x_l_stride = x.stride(-1);
80 + params.weight_c_stride = weight.stride(0);
81 + params.weight_width_stride = weight.stride(1);
82 + params.out_batch_stride = out.stride(0);
83 + params.out_c_stride = out.stride(1);
84 + params.out_l_stride = out.stride(-1);
85 +}
86 +
87 +
88 +at::Tensor
89 +causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight,
90 + const c10::optional<at::Tensor> &bias_,
91 + const c10::optional<at::Tensor> &seq_idx_,
92 + const c10::optional<at::Tensor> &initial_states_,
93 + const c10::optional<at::Tensor> &final_states_out_,
94 + bool silu_activation) {
95 + auto input_type = x.scalar_type();
96 + auto weight_type = weight.scalar_type();
97 + TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
98 + TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16);
99 +
100 + TORCH_CHECK(x.is_cuda());
101 + TORCH_CHECK(weight.is_cuda());
102 +
103 + const auto sizes = x.sizes();
104 + const int batch_size = sizes[0];
105 + const int dim = sizes[1];
106 + const int seqlen = sizes[2];
107 + const int width = weight.size(-1);
108 +
109 + CHECK_SHAPE(x, batch_size, dim, seqlen);
110 + CHECK_SHAPE(weight, dim, width);
111 +
112 + TORCH_CHECK(x.stride(2) == 1 || x.stride(1) == 1);
113 + const bool is_channel_last = x.stride(1) == 1 && x.stride(2) > 1;
114 +
115 + if (is_channel_last) {
116 + TORCH_CHECK(dim % 8 == 0, "causal_conv1d only supports channel dimension divisible by 8 for now");
117 + TORCH_CHECK(x.stride(2) % 8 == 0 and x.stride(0) % 8 == 0, "causal_conv1d with channel last layout requires strides (x.stride(0) and x.stride(2)) to be multiples of 8");
118 + }
119 + TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4");
120 +
121 + if (bias_.has_value()) {
122 + auto bias = bias_.value();
123 + TORCH_CHECK(bias.scalar_type() == weight_type);
124 + TORCH_CHECK(bias.is_cuda());
125 + TORCH_CHECK(bias.stride(-1) == 1);
126 + CHECK_SHAPE(bias, dim);
127 + }
128 +
129 + if (seq_idx_.has_value()) {
130 + TORCH_CHECK(is_channel_last, "seq_idx is only supported for channel last layout");
131 + auto seq_idx = seq_idx_.value();
132 + TORCH_CHECK(seq_idx.scalar_type() == torch::kInt32);
133 + TORCH_CHECK(seq_idx.is_cuda());
134 + TORCH_CHECK(seq_idx.is_contiguous());
135 + CHECK_SHAPE(seq_idx, batch_size, seqlen);
136 + }
137 +
138 + at::Tensor out = torch::empty_like(x);
139 +
140 + ConvParamsBase params;
141 + set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out,
142 + bias_.has_value() ? bias_.value().data_ptr() : nullptr,
143 + silu_activation);
144 +
145 + if (seq_idx_.has_value()) {
146 + params.seq_idx_ptr = seq_idx_.value().data_ptr();
147 + } else {
148 + params.seq_idx_ptr = nullptr;
149 + }
150 +
151 + if (initial_states_.has_value()) {
152 + TORCH_CHECK(is_channel_last, "initial_states is only supported for channel last layout");
153 + auto initial_states = initial_states_.value();
154 + TORCH_CHECK(initial_states.scalar_type() == input_type);
155 + TORCH_CHECK(initial_states.is_cuda());
156 + CHECK_SHAPE(initial_states, batch_size, dim, width - 1);
157 + TORCH_CHECK(initial_states.stride(1) == 1);
158 + params.initial_states_ptr = initial_states.data_ptr();
159 + params.initial_states_batch_stride = initial_states.stride(0);
160 + params.initial_states_c_stride = initial_states.stride(1);
161 + params.initial_states_l_stride = initial_states.stride(2);
162 + } else {
163 + params.initial_states_ptr = nullptr;
164 + }
165 +
166 + if (final_states_out_.has_value()) {
167 + TORCH_CHECK(is_channel_last, "final_states is only supported for channel last layout");
168 + auto final_states = final_states_out_.value();
169 + TORCH_CHECK(final_states.scalar_type() == input_type);
170 + TORCH_CHECK(final_states.is_cuda());
171 + CHECK_SHAPE(final_states, batch_size, dim, width - 1);
172 + TORCH_CHECK(final_states.stride(1) == 1);
173 + params.final_states_ptr = final_states.data_ptr();
174 + params.final_states_batch_stride = final_states.stride(0);
175 + params.final_states_c_stride = final_states.stride(1);
176 + params.final_states_l_stride = final_states.stride(2);
177 + } else {
178 + params.final_states_ptr = nullptr;
179 + }
180 +
181 + // Otherwise the kernel will be launched from cuda:0 device
182 + // Cast to char to avoid compiler warning about narrowing
183 + at::cuda::CUDAGuard device_guard{(char)x.get_device()};
184 + auto stream = at::cuda::getCurrentCUDAStream().stream();
185 + DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_fwd", [&] {
186 + if (!is_channel_last) {
187 + causal_conv1d_fwd_cuda<input_t, weight_t>(params, stream);
188 + } else {
189 + causal_conv1d_channellast_fwd_cuda<input_t, weight_t>(params, stream);
190 + }
191 + });
192 + return out;
193 +}
194 +
195 +
196 +at::Tensor
197 +causal_conv1d_update(const at::Tensor &x,
198 + const at::Tensor &conv_state,
199 + const at::Tensor &weight,
200 + const c10::optional<at::Tensor> &bias_,
201 + bool silu_activation) {
202 + auto input_type = x.scalar_type();
203 + auto weight_type = weight.scalar_type();
204 + TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
205 + TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16);
206 + TORCH_CHECK(weight_type == input_type, "weight type must equal to input type, other variations are disabled due to binary size limitations");
207 + TORCH_CHECK(conv_state.scalar_type() == input_type);
208 +
209 + TORCH_CHECK(x.is_cuda());
210 + TORCH_CHECK(conv_state.is_cuda());
211 + TORCH_CHECK(weight.is_cuda());
212 +
213 + const auto sizes = x.sizes();
214 + const int batch_size = sizes[0];
215 + const int dim = sizes[1];
216 + const int width = weight.size(-1);
217 +
218 + CHECK_SHAPE(x, batch_size, dim);
219 + CHECK_SHAPE(conv_state, batch_size, dim, width);
220 + CHECK_SHAPE(weight, dim, width);
221 +
222 + TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4");
223 +
224 + if (bias_.has_value()) {
225 + auto bias = bias_.value();
226 + TORCH_CHECK(bias.scalar_type() == weight_type);
227 + TORCH_CHECK(bias.is_cuda());
228 + TORCH_CHECK(bias.stride(-1) == 1);
229 + CHECK_SHAPE(bias, dim);
230 + }
231 +
232 + at::Tensor out = torch::empty_like(x);
233 +
234 + ConvParamsBase params;
235 + set_conv_params_fwd(params, batch_size, dim, /*seqlen=*/1, width, x, weight, out,
236 + bias_.has_value() ? bias_.value().data_ptr() : nullptr,
237 + silu_activation);
238 + params.conv_state_ptr = conv_state.data_ptr();
239 + // All stride are in elements, not bytes.
240 + params.conv_state_batch_stride = conv_state.stride(0);
241 + params.conv_state_c_stride = conv_state.stride(1);
242 + params.conv_state_l_stride = conv_state.stride(2);
243 +
244 + // Otherwise the kernel will be launched from cuda:0 device
245 + // Cast to char to avoid compiler warning about narrowing
246 + at::cuda::CUDAGuard device_guard{(char)x.get_device()};
247 + auto stream = at::cuda::getCurrentCUDAStream().stream();
248 + DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_update", [&] {
249 + causal_conv1d_update_cuda<input_t, weight_t>(params, stream);
250 + });
251 + return out;
252 +}
253 +
254 +template<int kNThreads_, int kWidth_, bool kIsVecLoad_, typename input_t_, typename weight_t_>
255 +struct Causal_conv1d_fwd_kernel_traits {
256 + using input_t = input_t_;
257 + using weight_t = weight_t_;
258 + static constexpr int kNThreads = kNThreads_;
259 + static constexpr int kWidth = kWidth_;
260 + static constexpr int kNBytes = sizeof(input_t);
261 + static_assert(kNBytes == 2 || kNBytes == 4);
262 + static constexpr int kNElts = kNBytes == 4 ? 4 : 8;
263 + static_assert(kWidth <= kNElts);
264 + static constexpr bool kIsVecLoad = kIsVecLoad_;
265 + using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
266 + using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNElts, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
267 + using BlockLoadVecT = cub::BlockLoad<vec_t, kNThreads, 1, cub::BLOCK_LOAD_DIRECT>;
268 + using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNElts, cub::BLOCK_STORE_WARP_TRANSPOSE>;
269 + using BlockStoreVecT = cub::BlockStore<vec_t, kNThreads, 1, cub::BLOCK_STORE_DIRECT>;
270 + static constexpr int kSmemIOSize = kIsVecLoad
271 + ? 0
272 + : custom_max({sizeof(typename BlockLoadT::TempStorage), sizeof(typename BlockStoreT::TempStorage)});
273 + static constexpr int kSmemExchangeSize = kNThreads * kNBytes * kNElts;
274 + static constexpr int kSmemSize = kSmemIOSize + kSmemExchangeSize;
275 +};
276 +
277 +template<typename Ktraits>
278 +__global__ __launch_bounds__(Ktraits::kNThreads)
279 +void causal_conv1d_fwd_kernel(ConvParamsBase params) {
280 + constexpr int kWidth = Ktraits::kWidth;
281 + constexpr int kNThreads = Ktraits::kNThreads;
282 + constexpr int kNElts = Ktraits::kNElts;
283 + static constexpr bool kIsVecLoad = Ktraits::kIsVecLoad;
284 + using input_t = typename Ktraits::input_t;
285 + using vec_t = typename Ktraits::vec_t;
286 + using weight_t = typename Ktraits::weight_t;
287 +
288 + // Shared memory.
289 + extern __shared__ char smem_[];
290 + auto& smem_load = reinterpret_cast<typename Ktraits::BlockLoadT::TempStorage&>(smem_);
291 + auto& smem_load_vec = reinterpret_cast<typename Ktraits::BlockLoadVecT::TempStorage&>(smem_);
292 + auto& smem_store = reinterpret_cast<typename Ktraits::BlockStoreT::TempStorage&>(smem_);
293 + auto& smem_store_vec = reinterpret_cast<typename Ktraits::BlockStoreVecT::TempStorage&>(smem_);
294 + vec_t *smem_exchange = reinterpret_cast<vec_t *>(smem_ + Ktraits::kSmemIOSize);
295 +
296 + const int tidx = threadIdx.x;
297 + const int batch_id = blockIdx.x;
298 + const int channel_id = blockIdx.y;
299 + input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
300 + + channel_id * params.x_c_stride;
301 + weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr) + channel_id * params.weight_c_stride;
302 + input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
303 + + channel_id * params.out_c_stride;
304 + float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[channel_id]);
305 +
306 + // Thread 0 will load the last elements of the previous chunk, so we initialize those to 0.
307 + if (tidx == 0) {
308 + input_t zeros[kNElts] = {0};
309 + smem_exchange[kNThreads - 1] = reinterpret_cast<vec_t *>(zeros)[0];
310 + }
311 +
312 + float weight_vals[kWidth];
313 + #pragma unroll
314 + for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); }
315 +
316 + constexpr int kChunkSize = kNThreads * kNElts;
317 + const int n_chunks = (params.seqlen + kChunkSize - 1) / kChunkSize;
318 + for (int chunk = 0; chunk < n_chunks; ++chunk) {
319 + input_t x_vals_load[2 * kNElts] = {0};
320 + if constexpr(kIsVecLoad) {
321 + typename Ktraits::BlockLoadVecT(smem_load_vec).Load(reinterpret_cast<vec_t*>(x), *reinterpret_cast<vec_t (*)[1]>(&x_vals_load[kNElts]), (params.seqlen - chunk * kChunkSize) / kNElts);
322 + } else {
323 + __syncthreads();
324 + typename Ktraits::BlockLoadT(smem_load).Load(x, *reinterpret_cast<input_t (*)[kNElts]>(&x_vals_load[kNElts]), params.seqlen - chunk * kChunkSize);
325 + }
326 + x += kChunkSize;
327 + __syncthreads();
328 + // Thread kNThreads - 1 don't write yet, so that thread 0 can read
329 + // the last elements of the previous chunk.
330 + if (tidx < kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast<vec_t *>(x_vals_load)[1]; }
331 + __syncthreads();
332 + reinterpret_cast<vec_t *>(x_vals_load)[0] = smem_exchange[tidx > 0 ? tidx - 1 : kNThreads - 1];
333 + __syncthreads();
334 + // Now thread kNThreads - 1 can write the last elements of the current chunk.
335 + if (tidx == kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast<vec_t *>(x_vals_load)[1]; }
336 +
337 + float x_vals[2 * kNElts];
338 + #pragma unroll
339 + for (int i = 0; i < 2 * kNElts; ++i) { x_vals[i] = float(x_vals_load[i]); }
340 +
341 + float out_vals[kNElts];
342 + #pragma unroll
343 + for (int i = 0; i < kNElts; ++i) {
344 + out_vals[i] = bias_val;
345 + #pragma unroll
346 + for (int w = 0; w < kWidth; ++w) {
347 + out_vals[i] += weight_vals[w] * x_vals[kNElts + i - (kWidth - w - 1)];
348 + }
349 + }
350 +
351 + if (params.silu_activation) {
352 + #pragma unroll
353 + for (int i = 0; i < kNElts; ++i) {
354 + out_vals[i] = out_vals[i] / (1 + expf(-out_vals[i]));
355 + }
356 + }
357 +
358 + input_t out_vals_store[kNElts];
359 + #pragma unroll
360 + for (int i = 0; i < kNElts; ++i) { out_vals_store[i] = out_vals[i]; }
361 + if constexpr(kIsVecLoad) {
362 + typename Ktraits::BlockStoreVecT(smem_store_vec).Store(reinterpret_cast<vec_t*>(out), reinterpret_cast<vec_t (&)[1]>(out_vals_store), (params.seqlen - chunk * kChunkSize) / kNElts);
363 + } else {
364 + typename Ktraits::BlockStoreT(smem_store).Store(out, out_vals_store, params.seqlen - chunk * kChunkSize);
365 + }
366 + out += kChunkSize;
367 + }
368 +}
369 +
370 +
371 +template<int kNThreads, int kWidth, typename input_t, typename weight_t>
372 +void causal_conv1d_fwd_launch(ConvParamsBase &params, cudaStream_t stream) {
373 + static constexpr int kNElts = sizeof(input_t) == 4 ? 4 : 8;
374 + BOOL_SWITCH(params.seqlen % kNElts == 0, kIsVecLoad, [&] {
375 + using Ktraits = Causal_conv1d_fwd_kernel_traits<kNThreads, kWidth, kIsVecLoad, input_t, weight_t>;
376 + constexpr int kSmemSize = Ktraits::kSmemSize;
377 + dim3 grid(params.batch, params.dim);
378 +
379 + auto kernel = &causal_conv1d_fwd_kernel<Ktraits>;
380 +
381 + if (kSmemSize >= 48 * 1024) {
382 + #ifndef USE_ROCM
383 + C10_CUDA_CHECK(cudaFuncSetAttribute(
384 + kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
385 + #else
386 + // There is a slight signature discrepancy in HIP and CUDA "FuncSetAttribute" function.
387 + C10_CUDA_CHECK(cudaFuncSetAttribute(
388 + (void *) kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
389 + std::cerr << "Warning (causal_conv1d fwd launch): attempting to set maxDynamicSharedMemorySize on an AMD GPU which is currently a non-op (in ROCm versions <= 6.1). This might lead to undefined behavior. \n" << std::endl;
390 + #endif
391 + }
392 + kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
393 +
394 + C10_CUDA_KERNEL_LAUNCH_CHECK();
395 + });
396 +}
397 +
398 +template<typename input_t, typename weight_t>
399 +void causal_conv1d_fwd_cuda(ConvParamsBase &params, cudaStream_t stream) {
400 + if (params.width == 2) {
401 + causal_conv1d_fwd_launch<128, 2, input_t, weight_t>(params, stream);
402 + } else if (params.width == 3) {
403 + causal_conv1d_fwd_launch<128, 3, input_t, weight_t>(params, stream);
404 + } else if (params.width == 4) {
405 + causal_conv1d_fwd_launch<128, 4, input_t, weight_t>(params, stream);
406 + }
407 +}
408 +
409 +template<int kNThreads_, int kWidth_, int kChunkSizeL_, bool kIsVecLoad_, typename input_t_, typename weight_t_>
410 +struct Causal_conv1d_channellast_fwd_kernel_traits {
411 + // The cache line is 128 bytes, and we try to read 16 bytes per thread.
412 + // So we have 8 threads per "row", so 32 or 64 elements in the channel dimension.
413 + // That leaves 4 columns per warp, and so 16 columns per block (assuming each block has 128
414 + // threads). Each each load is 16 x 32|64 elements in the L x C dimensions.
415 + using input_t = input_t_;
416 + using weight_t = weight_t_;
417 + static constexpr int kNThreads = kNThreads_;
418 + static_assert(kNThreads % 32 == 0);
419 + static constexpr int kNWarps = kNThreads / 32;
420 + static constexpr int kWidth = kWidth_;
421 + static constexpr int kChunkSizeL = kChunkSizeL_;
422 + static constexpr int kNBytes = sizeof(input_t);
423 + static_assert(kNBytes == 2 || kNBytes == 4);
424 + static constexpr int kNElts = kNBytes == 4 ? 4 : 8;
425 + static constexpr int kNEltsPerRow = 128 / kNBytes;
426 + static constexpr int kNThreadsPerRow = kNEltsPerRow / kNElts; // Always 8 for now
427 + static_assert(kNThreadsPerRow * kNBytes * kNElts == 128);
428 + static constexpr int kNColsPerWarp = 32 / kNThreadsPerRow; // Always 4 for now
429 + static_assert(kNColsPerWarp * kNThreadsPerRow == 32);
430 + static constexpr int kNColsPerLoad = kNColsPerWarp * kNWarps;
431 + static constexpr int kNLoads = kChunkSizeL / kNColsPerLoad;
432 + static_assert(kNLoads * kNColsPerLoad == kChunkSizeL);
433 + static constexpr bool kIsVecLoad = kIsVecLoad_;
434 + using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
435 + // using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
436 + // using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNItems, cub::BLOCK_STORE_WARP_TRANSPOSE>;
437 + // static constexpr int kSmemSize = std::max({sizeof(typename BlockLoadT::TempStorage),
438 + // sizeof(typename BlockStoreT::TempStorage)});
439 + // static constexpr int kSmemSize = kChunkSizeL * kNEltsPerRow * kNBytes;
440 +};
441 +
442 +template<typename Ktraits, bool kHasSeqIdx>
443 +__global__ __launch_bounds__(Ktraits::kNThreads)
444 +void causal_conv1d_channellast_fwd_kernel(ConvParamsBase params) {
445 + constexpr int kWidth = Ktraits::kWidth;
446 + constexpr int kNThreads = Ktraits::kNThreads;
447 + constexpr int kNElts = Ktraits::kNElts;
448 + constexpr int kNThreadsPerC = Ktraits::kNThreadsPerRow;
449 + constexpr int kLPerLoad = Ktraits::kNColsPerLoad;
450 + constexpr int kChunkSizeL = Ktraits::kChunkSizeL;
451 + constexpr int kChunkSizeC = Ktraits::kNEltsPerRow;
452 + using input_t = typename Ktraits::input_t;
453 + using vec_t = typename Ktraits::vec_t;
454 + using weight_t = typename Ktraits::weight_t;
455 +
456 + // Shared memory.
457 + __shared__ input_t x_smem[kWidth - 1 + kChunkSizeL][kChunkSizeC + kNElts];
458 +
459 + const int batch_id = blockIdx.x;
460 + const int chunk_l_id = blockIdx.y;
461 + const int chunk_c_id = blockIdx.z;
462 + const int tid = threadIdx.x;
463 + const int l_idx = tid / kNThreadsPerC;
464 + const int c_idx = tid % kNThreadsPerC;
465 + input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
466 + + (chunk_l_id * kChunkSizeL + l_idx) * params.x_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
467 + weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr)
468 + + chunk_c_id * kChunkSizeC * params.weight_c_stride;
469 + input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
470 + + (chunk_l_id * kChunkSizeL + l_idx) * params.out_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
471 + int *seq_idx = !kHasSeqIdx ? nullptr : reinterpret_cast<int *>(params.seq_idx_ptr)
472 + + batch_id * params.seqlen + chunk_l_id * kChunkSizeL;
473 + input_t *initial_states = params.initial_states_ptr == nullptr || chunk_l_id > 0 ? nullptr
474 + : reinterpret_cast<input_t *>(params.initial_states_ptr) + batch_id * params.initial_states_batch_stride + l_idx * params.initial_states_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
475 + // The last L-chunk will also have enough info to write to final states, since it also contain a few x values
476 + // from the previous L-chunk.
477 + input_t *final_states = params.final_states_ptr == nullptr || chunk_l_id < gridDim.y - 1 ? nullptr
478 + : reinterpret_cast<input_t *>(params.final_states_ptr) + batch_id * params.final_states_batch_stride + l_idx * params.final_states_l_stride + chunk_c_id * kChunkSizeC + c_idx * kNElts;
479 +
480 + #pragma unroll
481 + for (int l = 0; l < Ktraits::kNLoads; ++l) {
482 + input_t x_vals_load[kNElts] = {0};
483 + if (chunk_l_id * kChunkSizeL + l * kLPerLoad + l_idx < params.seqlen
484 + && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
485 + reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(x + l * kLPerLoad * params.x_l_stride);
486 + }
487 + reinterpret_cast<vec_t *>(x_smem[kWidth - 1 + l * kLPerLoad + l_idx])[c_idx] = reinterpret_cast<vec_t *>(x_vals_load)[0];
488 + }
489 + // Load the elements from the previous chunk that are needed for convolution.
490 + if (l_idx < kWidth - 1) {
491 + input_t x_vals_load[kNElts] = {0};
492 + if (chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) >= 0
493 + && chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) < params.seqlen
494 + && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
495 + reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(x - (kWidth - 1) * params.x_l_stride);
496 + } else if (initial_states != nullptr
497 + && chunk_l_id * kChunkSizeL + l_idx - (kWidth - 1) < 0
498 + && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
499 + reinterpret_cast<vec_t *>(x_vals_load)[0] = *reinterpret_cast<vec_t *>(initial_states);
500 + }
501 + reinterpret_cast<vec_t *>(x_smem[l_idx])[c_idx] = reinterpret_cast<vec_t *>(x_vals_load)[0];
502 + }
503 +
504 + __syncthreads();
505 +
506 + if (final_states != nullptr
507 + && l_idx < kWidth - 1
508 + && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
509 + // x_smem[0] contains element at index chunk_l_id * kChunkSizeL - (kWidth - 1)
510 + // So last few elements (index params.seqlen - kWidth + 1 + l_idx) are stored in x_smem[params.seqlen - kWidth + 1 + l_idx - (chunk_l_id * kChunkSizeL - kWidth + 1)][c_idx]
511 + *reinterpret_cast<vec_t *>(final_states) = reinterpret_cast<vec_t *>(x_smem[params.seqlen + l_idx - chunk_l_id * kChunkSizeL])[c_idx];
512 + }
513 +
514 + constexpr int kLPerThread = constexpr_min(kChunkSizeL * kChunkSizeC / kNThreads, kChunkSizeL);
515 + static_assert(kLPerThread * kNThreads == kChunkSizeL * kChunkSizeC);
516 + constexpr int kNThreadsPerRow = kChunkSizeL / kLPerThread;
517 + static_assert(kNThreadsPerRow * kLPerThread == kChunkSizeL);
518 + // kChunkSizeL, kLPerThread, kNThreadsPerRow should be powers of 2 for simplicity
519 + static_assert((kChunkSizeL & (kChunkSizeL - 1)) == 0);
520 + static_assert((kLPerThread & (kLPerThread - 1)) == 0);
521 + static_assert((kNThreadsPerRow & (kNThreadsPerRow - 1)) == 0);
522 + static_assert(kNThreadsPerRow <= 32);
523 +
524 + const int row_idx = tid / kNThreadsPerRow;
525 + const int col_idx = tid % kNThreadsPerRow;
526 +
527 + float bias_val = params.bias_ptr == nullptr || chunk_c_id * kChunkSizeC + row_idx >= params.dim ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[chunk_c_id * kChunkSizeC + row_idx]);
528 + float weight_vals[kWidth] = {0};
529 + if (chunk_c_id * kChunkSizeC + row_idx < params.dim) {
530 + #pragma unroll
531 + for (int w = 0; w < kWidth; ++w) {
532 + weight_vals[w] = weight[row_idx * params.weight_c_stride + w * params.weight_width_stride];
533 + }
534 + }
535 + float x_vals[kWidth - 1 + kLPerThread];
536 + #pragma unroll
537 + for (int i = 0; i < kWidth - 1 + kLPerThread; ++i) {
538 + x_vals[i] = float(x_smem[col_idx * kLPerThread + i][row_idx]);
539 + }
540 + int seq_idx_thread[kWidth - 1 + kLPerThread];
541 + if constexpr (kHasSeqIdx) {
542 + #pragma unroll
543 + for (int i = 0; i < kWidth - 1 + kLPerThread; ++i) {
544 + seq_idx_thread[i] = chunk_l_id * kChunkSizeL + col_idx * kLPerThread + i - (kWidth - 1) >= 0 ? seq_idx[col_idx * kLPerThread + i - (kWidth - 1)] : -1;
545 + }
546 + }
547 +
548 + float out_vals[kLPerThread];
549 + #pragma unroll
550 + for (int i = 0; i < kLPerThread; ++i) {
551 + out_vals[i] = bias_val;
552 + const int seq_idx_cur = !kHasSeqIdx ? 0 : seq_idx_thread[i + kWidth - 1];
553 + #pragma unroll
554 + for (int w = 0; w < kWidth; ++w) {
555 + if constexpr (!kHasSeqIdx) {
556 + out_vals[i] += weight_vals[w] * x_vals[i + w];
557 + } else {
558 + out_vals[i] += seq_idx_thread[i + w] == seq_idx_cur ? weight_vals[w] * x_vals[i + w] : 0.f;
559 + }
560 + }
561 + if (params.silu_activation) {out_vals[i] = out_vals[i] / (1 + expf(-out_vals[i])); }
562 + }
563 +
564 + __syncthreads();
565 + #pragma unroll
566 + for (int i = 0; i < kLPerThread; ++i) { x_smem[col_idx * kLPerThread + i][row_idx] = out_vals[i]; }
567 + __syncthreads();
568 +
569 + #pragma unroll
570 + for (int l = 0; l < Ktraits::kNLoads; ++l) {
571 + input_t out_vals_store[kNElts];
572 + reinterpret_cast<vec_t *>(out_vals_store)[0] = reinterpret_cast<vec_t *>(x_smem[l * kLPerLoad + l_idx])[c_idx];
573 + if (chunk_l_id * kChunkSizeL + l * kLPerLoad + l_idx < params.seqlen
574 + && chunk_c_id * kChunkSizeC + c_idx * kNElts < params.dim) {
575 + *reinterpret_cast<vec_t *>(out + l * kLPerLoad * params.out_l_stride) = reinterpret_cast<vec_t *>(out_vals_store)[0];
576 + }
577 + }
578 +
579 +}
580 +
581 +template<int kNThreads, int kWidth, typename input_t, typename weight_t>
582 +void causal_conv1d_channellast_fwd_launch(ConvParamsBase &params, cudaStream_t stream) {
583 + BOOL_SWITCH(params.seq_idx_ptr != nullptr, kHasSeqIdx, [&] {
584 + using Ktraits = Causal_conv1d_channellast_fwd_kernel_traits<kNThreads, kWidth, 64, true, input_t, weight_t>;
585 + // constexpr int kSmemSize = Ktraits::kSmemSize;
586 + constexpr int kChunkSizeL = Ktraits::kChunkSizeL;
587 + constexpr int kChunkSizeC = Ktraits::kNEltsPerRow;
588 + const int n_chunks_L = (params.seqlen + kChunkSizeL - 1) / kChunkSizeL;
589 + const int n_chunks_C = (params.dim + kChunkSizeC - 1) / kChunkSizeC;
590 + dim3 grid(params.batch, n_chunks_L, n_chunks_C);
591 + dim3 block(Ktraits::kNThreads);
592 + auto kernel = &causal_conv1d_channellast_fwd_kernel<Ktraits, kHasSeqIdx>;
593 + // if (kSmemSize >= 48 * 1024) {
594 + // C10_CUDA_CHECK(cudaFuncSetAttribute(
595 + // kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
596 + // }
597 + // kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
598 + kernel<<<grid, Ktraits::kNThreads, 0, stream>>>(params);
599 + C10_CUDA_KERNEL_LAUNCH_CHECK();
600 + });
601 +}
602 +
603 +template<typename input_t, typename weight_t>
604 +void causal_conv1d_channellast_fwd_cuda(ConvParamsBase &params, cudaStream_t stream) {
605 + if (params.width == 2) {
606 + causal_conv1d_channellast_fwd_launch<128, 2, input_t, weight_t>(params, stream);
607 + } else if (params.width == 3) {
608 + causal_conv1d_channellast_fwd_launch<128, 3, input_t, weight_t>(params, stream);
609 + } else if (params.width == 4) {
610 + causal_conv1d_channellast_fwd_launch<128, 4, input_t, weight_t>(params, stream);
611 + }
612 +}
613 +
614 +template void causal_conv1d_fwd_cuda<float, float>(ConvParamsBase &params, cudaStream_t stream);
615 +template void causal_conv1d_fwd_cuda<at::Half, at::Half>(ConvParamsBase &params, cudaStream_t stream);
616 +template void causal_conv1d_fwd_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase &params, cudaStream_t stream);
617 +
618 +template void causal_conv1d_channellast_fwd_cuda<float, float>(ConvParamsBase &params, cudaStream_t stream);
619 +template void causal_conv1d_channellast_fwd_cuda<at::Half, at::Half>(ConvParamsBase &params, cudaStream_t stream);
620 +template void causal_conv1d_channellast_fwd_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase &params, cudaStream_t stream);
621 +///////
622 +
623 +
624 +
625 +
626 +template<int kNThreads_, int kWidth_, typename input_t_, typename weight_t_>
627 +struct Causal_conv1d_update_kernel_traits {
628 + using input_t = input_t_;
629 + using weight_t = weight_t_;
630 + static constexpr int kNThreads = kNThreads_;
631 + static constexpr int kWidth = kWidth_;
632 + static constexpr int kNBytes = sizeof(input_t);
633 + static_assert(kNBytes == 2 || kNBytes == 4);
634 +};
635 +
636 +template<typename Ktraits>
637 +__global__ __launch_bounds__(Ktraits::kNThreads)
638 +void causal_conv1d_update_kernel(ConvParamsBase params) {
639 + constexpr int kWidth = Ktraits::kWidth;
640 + constexpr int kNThreads = Ktraits::kNThreads;
641 + using input_t = typename Ktraits::input_t;
642 + using weight_t = typename Ktraits::weight_t;
643 +
644 + const int tidx = threadIdx.x;
645 + const int batch_id = blockIdx.x;
646 + const int channel_id = blockIdx.y * kNThreads + tidx;
647 + input_t *x = reinterpret_cast<input_t *>(params.x_ptr) + batch_id * params.x_batch_stride
648 + + channel_id * params.x_c_stride;
649 + input_t *conv_state = reinterpret_cast<input_t *>(params.conv_state_ptr) + batch_id * params.conv_state_batch_stride
650 + + channel_id * params.conv_state_c_stride;
651 + weight_t *weight = reinterpret_cast<weight_t *>(params.weight_ptr) + channel_id * params.weight_c_stride;
652 + input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
653 + + channel_id * params.out_c_stride;
654 + float bias_val = params.bias_ptr == nullptr || channel_id >= params.dim ? 0.f : float(reinterpret_cast<weight_t *>(params.bias_ptr)[channel_id]);
655 +
656 + float weight_vals[kWidth] = {0};
657 + if (channel_id < params.dim) {
658 + #pragma unroll
659 + for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); }
660 + }
661 +
662 + float x_vals[kWidth] = {0};
663 + if (channel_id < params.dim) {
664 + #pragma unroll
665 + for (int i = 0; i < kWidth - 1; ++i) { x_vals[i] = float(conv_state[(i + 1) * params.conv_state_l_stride]); }
666 + x_vals[kWidth - 1] = float(x[0]);
667 + #pragma unroll
668 + for (int i = 0; i < kWidth; ++i) { conv_state[i * params.conv_state_l_stride] = input_t(x_vals[i]); }
669 + }
670 +
671 + float out_val = bias_val;
672 + #pragma unroll
673 + for (int i = 0; i < kWidth; ++i) { out_val += weight_vals[i] * x_vals[i]; }
674 + if (params.silu_activation) { out_val = out_val / (1 + expf(-out_val)); }
675 + if (channel_id < params.dim) { out[0] = input_t(out_val); }
676 +}
677 +
678 +template<int kNThreads, int kWidth, typename input_t, typename weight_t>
679 +void causal_conv1d_update_launch(ConvParamsBase &params, cudaStream_t stream) {
680 + using Ktraits = Causal_conv1d_update_kernel_traits<kNThreads, kWidth, input_t, weight_t>;
681 + dim3 grid(params.batch, (params.dim + kNThreads - 1) / kNThreads);
682 + auto kernel = &causal_conv1d_update_kernel<Ktraits>;
683 + kernel<<<grid, Ktraits::kNThreads, 0, stream>>>(params);
684 + C10_CUDA_KERNEL_LAUNCH_CHECK();
685 +}
686 +
687 +template<typename input_t, typename weight_t>
688 +void causal_conv1d_update_cuda(ConvParamsBase &params, cudaStream_t stream) {
689 + if (params.width == 2) {
690 + causal_conv1d_update_launch<64, 2, input_t, weight_t>(params, stream);
691 + } else if (params.width == 3) {
692 + causal_conv1d_update_launch<64, 3, input_t, weight_t>(params, stream);
693 + } else if (params.width == 4) {
694 + causal_conv1d_update_launch<64, 4, input_t, weight_t>(params, stream);
695 + }
696 +}
697 +
698 +template void causal_conv1d_update_cuda<float, float>(ConvParamsBase &params, cudaStream_t stream);
699 +template void causal_conv1d_update_cuda<at::Half, at::Half>(ConvParamsBase &params, cudaStream_t stream);
700 +template void causal_conv1d_update_cuda<at::BFloat16, at::BFloat16>(ConvParamsBase &params, cudaStream_t stream);
csrc/mamba/causal_conv1d/causal_conv1d.h new
+144
@@ -0,0 +1,144 @@
1 +/******************************************************************************
2 + * Copyright (c) 2024, Tri Dao.
3 + ******************************************************************************/
4 +// clang-format off
5 +// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d.h
6 +#pragma once
7 +
8 +#include <cuda_bf16.h>
9 +#include <cuda_fp16.h>
10 +////////////////////////////////////////////////////////////////////////////////////////////////////
11 +
12 +struct ConvParamsBase {
13 + using index_t = uint32_t;
14 +
15 + int batch, dim, seqlen, width;
16 + bool silu_activation;
17 +
18 + index_t x_batch_stride;
19 + index_t x_c_stride;
20 + index_t x_l_stride;
21 + index_t weight_c_stride;
22 + index_t weight_width_stride;
23 + index_t out_batch_stride;
24 + index_t out_c_stride;
25 + index_t out_l_stride;
26 +
27 + index_t conv_state_batch_stride;
28 + index_t conv_state_c_stride;
29 + index_t conv_state_l_stride;
30 +
31 + // Common data pointers.
32 + void *__restrict__ x_ptr;
33 + void *__restrict__ weight_ptr;
34 + void *__restrict__ bias_ptr;
35 + void *__restrict__ out_ptr;
36 +
37 + void *__restrict__ conv_state_ptr;
38 +
39 + void *__restrict__ seq_idx_ptr;
40 +
41 + // No __restrict__ since initial_states could be the same as final_states.
42 + void * initial_states_ptr;
43 + index_t initial_states_batch_stride;
44 + index_t initial_states_l_stride;
45 + index_t initial_states_c_stride;
46 +
47 + void * final_states_ptr;
48 + index_t final_states_batch_stride;
49 + index_t final_states_l_stride;
50 + index_t final_states_c_stride;
51 +};
52 +
53 +
54 +#ifndef USE_ROCM
55 + #include <cuda_bf16.h>
56 +
57 + template<typename T>
58 + __device__ inline T shuffle_xor(T val, int offset) {
59 + return __shfl_xor_sync(uint32_t(-1), val, offset);
60 + }
61 +
62 + constexpr size_t custom_max(std::initializer_list<size_t> ilist)
63 + {
64 + return std::max(ilist);
65 + }
66 +
67 + template<typename T>
68 + constexpr T constexpr_min(T a, T b) {
69 + return std::min(a, b);
70 + }
71 +
72 +#else
73 + #include <hip/hip_bf16.h>
74 +
75 + template<typename T>
76 + __device__ inline T shuffle_xor(T val, int offset) {
77 + return __shfl_xor(val, offset);
78 + }
79 + constexpr size_t custom_max(std::initializer_list<size_t> ilist)
80 + {
81 + return *std::max_element(ilist.begin(), ilist.end());
82 + }
83 +
84 + template<typename T>
85 + constexpr T constexpr_min(T a, T b) {
86 + return a < b ? a : b;
87 + }
88 +#endif
89 +
90 +////////////////////////////////////////////////////////////////////////////////////////////////////
91 +
92 +template<int BYTES> struct BytesToType {};
93 +
94 +template<> struct BytesToType<16> {
95 + using Type = uint4;
96 + static_assert(sizeof(Type) == 16);
97 +};
98 +
99 +template<> struct BytesToType<8> {
100 + using Type = uint64_t;
101 + static_assert(sizeof(Type) == 8);
102 +};
103 +
104 +template<> struct BytesToType<4> {
105 + using Type = uint32_t;
106 + static_assert(sizeof(Type) == 4);
107 +};
108 +
109 +template<> struct BytesToType<2> {
110 + using Type = uint16_t;
111 + static_assert(sizeof(Type) == 2);
112 +};
113 +
114 +template<> struct BytesToType<1> {
115 + using Type = uint8_t;
116 + static_assert(sizeof(Type) == 1);
117 +};
118 +
119 +////////////////////////////////////////////////////////////////////////////////////////////////////
120 +
121 +template<typename T>
122 +struct SumOp {
123 +__device__ inline T operator()(T const & x, T const & y) { return x + y; }
124 +};
125 +
126 +template<int THREADS>
127 +struct Allreduce {
128 + static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4);
129 + template<typename T, typename Operator>
130 + static __device__ inline T run(T x, Operator &op) {
131 + constexpr int OFFSET = THREADS / 2;
132 + x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET));
133 + return Allreduce<OFFSET>::run(x, op);
134 + }
135 +};
136 +
137 +template<>
138 +struct Allreduce<2> {
139 +template<typename T, typename Operator>
140 +static __device__ inline T run(T x, Operator &op) {
141 + x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1));
142 + return x;
143 +}
144 +};
csrc/mamba/causal_conv1d/static_switch.h new
+28
@@ -0,0 +1,28 @@
1 +// Inspired by
2 +// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h
3 +// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h
4 +// clang-format off
5 +// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/static_switch.h
6 +
7 +#pragma once
8 +
9 +/// @param COND - a boolean expression to switch by
10 +/// @param CONST_NAME - a name given for the constexpr bool variable.
11 +/// @param ... - code to execute for true and false
12 +///
13 +/// Usage:
14 +/// ```
15 +/// BOOL_SWITCH(flag, BoolConst, [&] {
16 +/// some_function<BoolConst>(...);
17 +/// });
18 +/// ```
19 +#define BOOL_SWITCH(COND, CONST_NAME, ...) \
20 + [&] { \
21 + if (COND) { \
22 + static constexpr bool CONST_NAME = true; \
23 + return __VA_ARGS__(); \
24 + } else { \
25 + static constexpr bool CONST_NAME = false; \
26 + return __VA_ARGS__(); \
27 + } \
28 + }()
csrc/mamba/mamba_ssm/selective_scan.h new
+276
@@ -0,0 +1,276 @@
1 +/******************************************************************************
2 + * Copyright (c) 2023, Tri Dao.
3 + ******************************************************************************/
4 +// clang-format off
5 +// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/selective_scan.h
6 +
7 +#pragma once
8 +
9 +#ifndef USE_ROCM
10 + #include <cuda_bf16.h>
11 +#else
12 + #include <hip/hip_bf16.h>
13 +#endif
14 +#include <cuda_fp16.h>
15 +////////////////////////////////////////////////////////////////////////////////////////////////////
16 +
17 +struct SSMParamsBase {
18 + using index_t = uint32_t;
19 +
20 + int batch, dim, seqlen, dstate, n_groups, n_chunks;
21 + int dim_ngroups_ratio;
22 + bool is_variable_B;
23 + bool is_variable_C;
24 +
25 + bool delta_softplus;
26 +
27 + index_t A_d_stride;
28 + index_t A_dstate_stride;
29 + index_t B_batch_stride;
30 + index_t B_d_stride;
31 + index_t B_dstate_stride;
32 + index_t B_group_stride;
33 + index_t C_batch_stride;
34 + index_t C_d_stride;
35 + index_t C_dstate_stride;
36 + index_t C_group_stride;
37 + index_t u_batch_stride;
38 + index_t u_d_stride;
39 + index_t delta_batch_stride;
40 + index_t delta_d_stride;
41 + index_t z_batch_stride;
42 + index_t z_d_stride;
43 + index_t out_batch_stride;
44 + index_t out_d_stride;
45 + index_t out_z_batch_stride;
46 + index_t out_z_d_stride;
47 +
48 + // Common data pointers.
49 + void *__restrict__ A_ptr;
50 + void *__restrict__ B_ptr;
51 + void *__restrict__ C_ptr;
52 + void *__restrict__ D_ptr;
53 + void *__restrict__ u_ptr;
54 + void *__restrict__ delta_ptr;
55 + void *__restrict__ delta_bias_ptr;
56 + void *__restrict__ out_ptr;
57 + void *__restrict__ x_ptr;
58 + void *__restrict__ z_ptr;
59 + void *__restrict__ out_z_ptr;
60 + void *__restrict__ index_ptr;
61 +};
62 +
63 +
64 +
65 +
66 +#ifndef USE_ROCM
67 +
68 + constexpr size_t custom_max(std::initializer_list<size_t> ilist)
69 + {
70 + return std::max(ilist);
71 + }
72 +
73 + template<typename T>
74 + constexpr T constexpr_min(T a, T b) {
75 + return std::min(a, b);
76 + }
77 +
78 +#else
79 + constexpr size_t custom_max(std::initializer_list<size_t> ilist)
80 + {
81 + return *std::max_element(ilist.begin(), ilist.end());
82 + }
83 +
84 + template<typename T>
85 + constexpr T constexpr_min(T a, T b) {
86 + return a < b ? a : b;
87 + }
88 +#endif
89 +
90 +
91 +#define MAX_DSTATE 256
92 +
93 +
94 +inline __device__ float2 operator+(const float2 & a, const float2 & b){
95 + return {a.x + b.x, a.y + b.y};
96 +}
97 +
98 +inline __device__ float3 operator+(const float3 &a, const float3 &b) {
99 + return {a.x + b.x, a.y + b.y, a.z + b.z};
100 +}
101 +
102 +inline __device__ float4 operator+(const float4 & a, const float4 & b){
103 + return {a.x + b.x, a.y + b.y, a.z + b.z, a.w + b.w};
104 +}
105 +
106 +////////////////////////////////////////////////////////////////////////////////////////////////////
107 +
108 +template<int BYTES> struct BytesToType {};
109 +
110 +template<> struct BytesToType<16> {
111 + using Type = uint4;
112 + static_assert(sizeof(Type) == 16);
113 +};
114 +
115 +template<> struct BytesToType<8> {
116 + using Type = uint64_t;
117 + static_assert(sizeof(Type) == 8);
118 +};
119 +
120 +template<> struct BytesToType<4> {
121 + using Type = uint32_t;
122 + static_assert(sizeof(Type) == 4);
123 +};
124 +
125 +template<> struct BytesToType<2> {
126 + using Type = uint16_t;
127 + static_assert(sizeof(Type) == 2);
128 +};
129 +
130 +template<> struct BytesToType<1> {
131 + using Type = uint8_t;
132 + static_assert(sizeof(Type) == 1);
133 +};
134 +
135 +////////////////////////////////////////////////////////////////////////////////////////////////////
136 +
137 +template<typename scalar_t, int N>
138 +struct Converter{
139 + static inline __device__ void to_float(const scalar_t (&src)[N], float (&dst)[N]) {
140 + #pragma unroll
141 + for (int i = 0; i < N; ++i) { dst[i] = src[i]; }
142 + }
143 +};
144 +
145 +template<int N>
146 +struct Converter<at::Half, N>{
147 + static inline __device__ void to_float(const at::Half (&src)[N], float (&dst)[N]) {
148 + static_assert(N % 2 == 0);
149 + auto &src2 = reinterpret_cast<const half2 (&)[N / 2]>(src);
150 + auto &dst2 = reinterpret_cast<float2 (&)[N / 2]>(dst);
151 + #pragma unroll
152 + for (int i = 0; i < N / 2; ++i) { dst2[i] = __half22float2(src2[i]); }
153 + }
154 +};
155 +
156 +#if __CUDA_ARCH__ >= 800
157 +template<int N>
158 +struct Converter<at::BFloat16, N>{
159 + static inline __device__ void to_float(const at::BFloat16 (&src)[N], float (&dst)[N]) {
160 + static_assert(N % 2 == 0);
161 + auto &src2 = reinterpret_cast<const nv_bfloat162 (&)[N / 2]>(src);
162 + auto &dst2 = reinterpret_cast<float2 (&)[N / 2]>(dst);
163 + #pragma unroll
164 + for (int i = 0; i < N / 2; ++i) { dst2[i] = __bfloat1622float2(src2[i]); }
165 + }
166 +};
167 +#endif
168 +
169 +////////////////////////////////////////////////////////////////////////////////////////////////////
170 +
171 +
172 +template<typename scalar_t> struct SSMScanOp;
173 +
174 +template<>
175 +struct SSMScanOp<float> {
176 + __device__ __forceinline__ float2 operator()(const float2 &ab0, const float2 &ab1) const {
177 + return make_float2(ab1.x * ab0.x, ab1.x * ab0.y + ab1.y);
178 + }
179 +};
180 +
181 +// A stateful callback functor that maintains a running prefix to be applied
182 +// during consecutive scan operations.
183 +template <typename scalar_t> struct SSMScanPrefixCallbackOp {
184 + using scan_t = std::conditional_t<std::is_same_v<scalar_t, float>, float2, float4>;
185 + scan_t running_prefix;
186 + // Constructor
187 + __device__ SSMScanPrefixCallbackOp(scan_t running_prefix_) : running_prefix(running_prefix_) {}
188 + // Callback operator to be entered by the first warp of threads in the block.
189 + // Thread-0 is responsible for returning a value for seeding the block-wide scan.
190 + __device__ scan_t operator()(scan_t block_aggregate) {
191 + scan_t old_prefix = running_prefix;
192 + running_prefix = SSMScanOp<scalar_t>()(running_prefix, block_aggregate);
193 + return old_prefix;
194 + }
195 +};
196 +
197 +////////////////////////////////////////////////////////////////////////////////////////////////////
198 +
199 +template<typename Ktraits>
200 +inline __device__ void load_input(typename Ktraits::input_t *u,
201 + typename Ktraits::input_t (&u_vals)[Ktraits::kNItems],
202 + typename Ktraits::BlockLoadT::TempStorage &smem_load,
203 + int seqlen) {
204 + if constexpr (Ktraits::kIsEvenLen) {
205 + auto& smem_load_vec = reinterpret_cast<typename Ktraits::BlockLoadVecT::TempStorage&>(smem_load);
206 + using vec_t = typename Ktraits::vec_t;
207 + typename Ktraits::BlockLoadVecT(smem_load_vec).Load(
208 + reinterpret_cast<vec_t*>(u),
209 + reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(u_vals)
210 + #ifdef USE_ROCM
211 + , Ktraits::kNThreads * Ktraits::kNLoads
212 + #endif
213 +
214 + );
215 + } else {
216 + typename Ktraits::BlockLoadT(smem_load).Load(u, u_vals, seqlen, 0.f);
217 + }
218 +}
219 +
220 +template<typename Ktraits>
221 +inline __device__ void load_index(int *u,
222 + int (&u_vals)[Ktraits::kNItems],
223 + typename Ktraits::BlockLoadIndexT::TempStorage &smem_load_index,
224 + int seqlen) {
225 + if constexpr (Ktraits::kIsEvenLen) {
226 + auto& smem_load_index_vec = reinterpret_cast<typename Ktraits::BlockLoadIndexVecT::TempStorage&>(smem_load_index);
227 + Ktraits::BlockLoadIndexVecT(smem_load_index_vec).Load(
228 + reinterpret_cast<uint4*>(u),
229 + reinterpret_cast<uint4(&)[Ktraits::kNLoadsIndex]>(u_vals)
230 + );
231 + } else {
232 + Ktraits::BlockLoadIndexT(smem_load_index).Load(u, u_vals, seqlen, 0);
233 + }
234 +}
235 +
236 +template<typename Ktraits>
237 +inline __device__ void load_weight(typename Ktraits::input_t *Bvar,
238 + typename Ktraits::weight_t (&B_vals)[Ktraits::kNItems],
239 + typename Ktraits::BlockLoadWeightT::TempStorage &smem_load_weight,
240 + int seqlen) {
241 + constexpr int kNItems = Ktraits::kNItems;
242 + typename Ktraits::input_t B_vals_load[kNItems];
243 + if constexpr (Ktraits::kIsEvenLen) {
244 + auto& smem_load_weight_vec = reinterpret_cast<typename Ktraits::BlockLoadWeightVecT::TempStorage&>(smem_load_weight);
245 + using vec_t = typename Ktraits::vec_t;
246 + typename Ktraits::BlockLoadWeightVecT(smem_load_weight_vec).Load(
247 + reinterpret_cast<vec_t*>(Bvar),
248 + reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(B_vals_load)
249 + );
250 + } else {
251 + typename Ktraits::BlockLoadWeightT(smem_load_weight).Load(Bvar, B_vals_load, seqlen, 0.f);
252 + }
253 + // #pragma unroll
254 + // for (int i = 0; i < kNItems; ++i) { B_vals[i] = B_vals_load[i]; }
255 + Converter<typename Ktraits::input_t, kNItems>::to_float(B_vals_load, B_vals);
256 +}
257 +
258 +template<typename Ktraits>
259 +inline __device__ void store_output(typename Ktraits::input_t *out,
260 + const float (&out_vals)[Ktraits::kNItems],
261 + typename Ktraits::BlockStoreT::TempStorage &smem_store,
262 + int seqlen) {
263 + typename Ktraits::input_t write_vals[Ktraits::kNItems];
264 + #pragma unroll
265 + for (int i = 0; i < Ktraits::kNItems; ++i) { write_vals[i] = out_vals[i]; }
266 + if constexpr (Ktraits::kIsEvenLen) {
267 + auto& smem_store_vec = reinterpret_cast<typename Ktraits::BlockStoreVecT::TempStorage&>(smem_store);
268 + using vec_t = typename Ktraits::vec_t;
269 + typename Ktraits::BlockStoreVecT(smem_store_vec).Store(
270 + reinterpret_cast<vec_t*>(out),
271 + reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(write_vals)
272 + );
273 + } else {
274 + typename Ktraits::BlockStoreT(smem_store).Store(out, write_vals, seqlen);
275 + }
276 +}
csrc/mamba/mamba_ssm/selective_scan_fwd.cu new
+593
@@ -0,0 +1,593 @@
1 +// clang-format off
2 +// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/selective_scan_fwd_kernel.cuh
3 +#include <torch/all.h>
4 +#include <ATen/cuda/CUDAContext.h>
5 +#include <c10/cuda/CUDAGuard.h>
6 +#include "selective_scan.h"
7 +
8 +#include <c10/util/BFloat16.h>
9 +#include <c10/util/Half.h>
10 +#include <c10/cuda/CUDAException.h> // For C10_CUDA_CHECK and C10_CUDA_KERNEL_LAUNCH_CHECK
11 +
12 +#ifndef USE_ROCM
13 + #include <cub/block/block_load.cuh>
14 + #include <cub/block/block_store.cuh>
15 + #include <cub/block/block_scan.cuh>
16 +#else
17 + #include <hipcub/hipcub.hpp>
18 + namespace cub = hipcub;
19 +#endif
20 +
21 +#include "selective_scan.h"
22 +#include "static_switch.h"
23 +
24 +template<int kNThreads_, int kNItems_, int kNRows_, bool kIsEvenLen_,
25 + bool kIsVariableB_, bool kIsVariableC_,
26 + bool kHasZ_, bool kUseIndex_, typename input_t_, typename weight_t_>
27 +struct Selective_Scan_fwd_kernel_traits {
28 + static_assert(kNItems_ % 4 == 0);
29 + using input_t = input_t_;
30 + using weight_t = weight_t_;
31 + static constexpr int kNThreads = kNThreads_;
32 + // Setting MinBlocksPerMP to be 3 (instead of 2) for 128 threads improves occupancy.
33 + static constexpr int kMinBlocks = kNThreads < 128 ? 5 : 3;
34 + static constexpr int kNItems = kNItems_;
35 + static constexpr int kNRows = kNRows_;
36 + static constexpr int kNBytes = sizeof(input_t);
37 + static_assert(kNBytes == 2 || kNBytes == 4);
38 + static constexpr int kNElts = kNBytes == 4 ? 4 : constexpr_min(8, kNItems);
39 + static_assert(kNItems % kNElts == 0);
40 + static constexpr int kNLoads = kNItems / kNElts;
41 + static constexpr bool kIsEvenLen = kIsEvenLen_;
42 + static constexpr bool kIsVariableB = kIsVariableB_;
43 + static constexpr bool kIsVariableC = kIsVariableC_;
44 + static constexpr bool kHasZ = kHasZ_;
45 + static constexpr bool kUseIndex = kUseIndex_;
46 +
47 + static constexpr bool kDirectIO = kIsEvenLen && kNLoads == 1;
48 + static constexpr int kNLoadsIndex = kNItems / 4;
49 + using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
50 + using scan_t = float2;
51 + using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
52 + using BlockLoadVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads,
53 + !kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
54 + using BlockLoadIndexT = cub::BlockLoad<int, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
55 + using BlockLoadIndexVecT = cub::BlockLoad<uint4, kNThreads, kNLoadsIndex,
56 + !(kIsEvenLen && kNLoadsIndex == 1) ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
57 + using BlockLoadWeightT = cub::BlockLoad<input_t, kNThreads, kNItems , cub::BLOCK_LOAD_WARP_TRANSPOSE>;
58 + using BlockLoadWeightVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads ,
59 + !kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
60 + using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNItems, cub::BLOCK_STORE_WARP_TRANSPOSE>;
61 + using BlockStoreVecT = cub::BlockStore<vec_t, kNThreads, kNLoads,
62 + !kDirectIO ? cub::BLOCK_STORE_WARP_TRANSPOSE : cub::BLOCK_STORE_DIRECT>;
63 + // using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_RAKING_MEMOIZE>;
64 + // using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_RAKING>;
65 + using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_WARP_SCANS>;
66 + static constexpr int kSmemIOSize = custom_max({sizeof(typename BlockLoadT::TempStorage),
67 + sizeof(typename BlockLoadVecT::TempStorage),
68 + sizeof(typename BlockLoadIndexT::TempStorage),
69 + sizeof(typename BlockLoadIndexVecT::TempStorage),
70 + (int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightT::TempStorage),
71 + (int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightVecT::TempStorage),
72 + sizeof(typename BlockStoreT::TempStorage),
73 + sizeof(typename BlockStoreVecT::TempStorage)});
74 + static constexpr int kSmemSize = kSmemIOSize + sizeof(typename BlockScanT::TempStorage);
75 +};
76 +
77 +template<typename Ktraits>
78 +__global__ __launch_bounds__(Ktraits::kNThreads, Ktraits::kMinBlocks)
79 +void selective_scan_fwd_kernel(SSMParamsBase params) {
80 + constexpr bool kIsVariableB = Ktraits::kIsVariableB;
81 + constexpr bool kIsVariableC = Ktraits::kIsVariableC;
82 + constexpr bool kHasZ = Ktraits::kHasZ;
83 + constexpr bool kUseIndex = Ktraits::kUseIndex;
84 + constexpr int kNThreads = Ktraits::kNThreads;
85 + constexpr int kNItems = Ktraits::kNItems;
86 + constexpr int kNRows = Ktraits::kNRows;
87 + constexpr bool kDirectIO = Ktraits::kDirectIO;
88 + using input_t = typename Ktraits::input_t;
89 + using weight_t = typename Ktraits::weight_t;
90 + using scan_t = typename Ktraits::scan_t;
91 +
92 + // Shared memory.
93 + extern __shared__ char smem_[];
94 + // cast to lvalue reference of expected type
95 + // char *smem_loadstorescan = smem_ + 2 * MAX_DSTATE * sizeof(weight_t);
96 + // auto& smem_load = reinterpret_cast<typename BlockLoadT::TempStorage&>(smem_ + 2 * MAX_DSTATE * sizeof(weight_t));
97 + // auto& smem_load = reinterpret_cast<typename BlockLoadT::TempStorage&>(smem_loadstorescan);
98 + auto& smem_load = reinterpret_cast<typename Ktraits::BlockLoadT::TempStorage&>(smem_);
99 + auto& smem_load_weight = reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage&>(smem_);
100 + auto& smem_load_index = reinterpret_cast<typename Ktraits::BlockLoadIndexT::TempStorage&>(smem_);
101 + auto& smem_load_weight1 = *reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage*>(smem_ + sizeof(typename Ktraits::BlockLoadWeightT::TempStorage));
102 + auto& smem_store = reinterpret_cast<typename Ktraits::BlockStoreT::TempStorage&>(smem_);
103 + auto& smem_scan = *reinterpret_cast<typename Ktraits::BlockScanT::TempStorage*>(smem_ + Ktraits::kSmemIOSize);
104 + // weight_t *smem_a = reinterpret_cast<weight_t *>(smem_ + smem_loadstorescan_size);
105 + // weight_t *smem_bc = reinterpret_cast<weight_t *>(smem_a + MAX_DSTATE);
106 + scan_t *smem_running_prefix = reinterpret_cast<scan_t *>(smem_ + Ktraits::kSmemSize);
107 +
108 + const int batch_id = blockIdx.x;
109 + const int dim_id = blockIdx.y;
110 + const int group_id = dim_id / (params.dim_ngroups_ratio);
111 + input_t *u = reinterpret_cast<input_t *>(params.u_ptr) + batch_id * params.u_batch_stride
112 + + dim_id * kNRows * params.u_d_stride;
113 + input_t *delta = reinterpret_cast<input_t *>(params.delta_ptr) + batch_id * params.delta_batch_stride
114 + + dim_id * kNRows * params.delta_d_stride;
115 + weight_t *A = reinterpret_cast<weight_t *>(params.A_ptr) + dim_id * kNRows * params.A_d_stride;
116 + weight_t *B = reinterpret_cast<weight_t *>(params.B_ptr) + dim_id * kNRows * params.B_d_stride;
117 + input_t *Bvar = reinterpret_cast<input_t *>(params.B_ptr) + batch_id * params.B_batch_stride + group_id * params.B_group_stride;
118 + weight_t *C = reinterpret_cast<weight_t *>(params.C_ptr) + dim_id * kNRows * params.C_d_stride;
119 + input_t *Cvar = reinterpret_cast<input_t *>(params.C_ptr) + batch_id * params.C_batch_stride + group_id * params.C_group_stride;
120 + scan_t *x = reinterpret_cast<scan_t *>(params.x_ptr) + (batch_id * params.dim + dim_id * kNRows) * params.n_chunks * params.dstate;
121 + int *index = !kUseIndex ? nullptr :reinterpret_cast<int *>(params.index_ptr) + batch_id * params.seqlen;
122 +
123 + float D_val[kNRows] = {0};
124 + if (params.D_ptr != nullptr) {
125 + #pragma unroll
126 + for (int r = 0; r < kNRows; ++r) {
127 + D_val[r] = reinterpret_cast<float *>(params.D_ptr)[dim_id * kNRows + r];
128 + }
129 + }
130 + float delta_bias[kNRows] = {0};
131 + if (params.delta_bias_ptr != nullptr) {
132 + #pragma unroll
133 + for (int r = 0; r < kNRows; ++r) {
134 + delta_bias[r] = reinterpret_cast<float *>(params.delta_bias_ptr)[dim_id * kNRows + r];
135 + }
136 + }
137 +
138 +
139 + // for (int state_idx = threadIdx.x; state_idx < params.dstate; state_idx += blockDim.x) {
140 + // smem_a[state_idx] = A[state_idx * params.A_dstate_stride];
141 + // smem_bc[state_idx] = B[state_idx * params.B_dstate_stride] * C[state_idx * params.C_dstate_stride];
142 + // }
143 +
144 + constexpr int kChunkSize = kNThreads * kNItems;
145 + for (int chunk = 0; chunk < params.n_chunks; ++chunk) {
146 + input_t u_vals[kNRows][kNItems], delta_vals_load[kNRows][kNItems];
147 + int index_vals_load[kNRows][kNItems];
148 +
149 + __syncthreads();
150 + #pragma unroll
151 + for (int r = 0; r < kNRows; ++r) {
152 + if constexpr (!kDirectIO) {
153 + if (r > 0) { __syncthreads(); }
154 + }
155 + load_input<Ktraits>(u + r * params.u_d_stride, u_vals[r], smem_load, params.seqlen - chunk * kChunkSize);
156 + if constexpr (!kDirectIO) { __syncthreads(); }
157 + load_input<Ktraits>(delta + r * params.delta_d_stride, delta_vals_load[r], smem_load, params.seqlen - chunk * kChunkSize);
158 + if constexpr (kUseIndex) {
159 + load_index<Ktraits>(index + r * params.delta_d_stride, index_vals_load[r], smem_load_index, params.seqlen - chunk * kChunkSize);
160 + }
161 + }
162 + if constexpr (kUseIndex) {
163 + index += kChunkSize;
164 + }
165 + u += kChunkSize;
166 + delta += kChunkSize;
167 +
168 + float delta_vals[kNRows][kNItems], delta_u_vals[kNRows][kNItems], out_vals[kNRows][kNItems];
169 + #pragma unroll
170 + for (int r = 0; r < kNRows; ++r) {
171 + #pragma unroll
172 + for (int i = 0; i < kNItems; ++i) {
173 + float u_val = float(u_vals[r][i]);
174 + delta_vals[r][i] = float(delta_vals_load[r][i]) + delta_bias[r];
175 + if (params.delta_softplus) {
176 + delta_vals[r][i] = delta_vals[r][i] <= 20.f ? log1pf(expf(delta_vals[r][i])) : delta_vals[r][i];
177 + }
178 + delta_u_vals[r][i] = delta_vals[r][i] * u_val;
179 + out_vals[r][i] = D_val[r] * u_val;
180 + }
181 + }
182 +
183 + __syncthreads();
184 + for (int state_idx = 0; state_idx < params.dstate; ++state_idx) {
185 + weight_t A_val[kNRows];
186 + #pragma unroll
187 + for (int r = 0; r < kNRows; ++r) {
188 + A_val[r] = A[state_idx * params.A_dstate_stride + r * params.A_d_stride];
189 + // Multiply the real part of A with LOG2E so we can use exp2f instead of expf.
190 + constexpr float kLog2e = M_LOG2E;
191 + A_val[r] *= kLog2e;
192 + }
193 + // This variable holds B * C if both B and C are constant across seqlen. If only B varies
194 + // across seqlen, this holds C. If only C varies across seqlen, this holds B.
195 + // If both B and C vary, this is unused.
196 + weight_t BC_val[kNRows];
197 + weight_t B_vals[kNItems], C_vals[kNItems];
198 + if constexpr (kIsVariableB) {
199 + load_weight<Ktraits>(Bvar + state_idx * params.B_dstate_stride, B_vals,
200 + smem_load_weight, (params.seqlen - chunk * kChunkSize) * (1));
201 + if constexpr (!kIsVariableC) {
202 + #pragma unroll
203 + for (int r = 0; r < kNRows; ++r) {
204 + BC_val[r] = C[state_idx * params.C_dstate_stride + r * params.C_d_stride];
205 + }
206 + }
207 + }
208 + if constexpr (kIsVariableC) {
209 + auto &smem_load_weight_C = !kIsVariableB ? smem_load_weight : smem_load_weight1;
210 + load_weight<Ktraits>(Cvar + state_idx * params.C_dstate_stride, C_vals,
211 + smem_load_weight_C, (params.seqlen - chunk * kChunkSize) * (1 ));
212 + if constexpr (!kIsVariableB) {
213 + #pragma unroll
214 + for (int r = 0; r < kNRows; ++r) {
215 + BC_val[r] = B[state_idx * params.B_dstate_stride + r * params.B_d_stride];
216 + }
217 + }
218 + }
219 + if constexpr (!kIsVariableB && !kIsVariableC) {
220 + #pragma unroll
221 + for (int r = 0; r < kNRows; ++r) {
222 + BC_val[r] = B[state_idx * params.B_dstate_stride + r * params.B_d_stride] * C[state_idx * params.C_dstate_stride + r * params.C_d_stride];
223 + }
224 + }
225 +
226 + #pragma unroll
227 + for (int r = 0; r < kNRows; ++r) {
228 + if (r > 0) { __syncthreads(); } // Scan could be using the same smem
229 + scan_t thread_data[kNItems];
230 + #pragma unroll
231 + for (int i = 0; i < kNItems; ++i) {
232 + thread_data[i] = make_float2(exp2f(delta_vals[r][i] * A_val[r]),
233 + !kIsVariableB ? delta_u_vals[r][i] : B_vals[i] * delta_u_vals[r][i]);
234 +
235 + // Reset A bar for cumulative sequences (Real)
236 + if constexpr (kUseIndex) {
237 + if (index_vals_load[r][i] == 0) {
238 + thread_data[i].x = 0.f;
239 + }
240 + }
241 +
242 + if constexpr (!Ktraits::kIsEvenLen) { // So that the last state is correct
243 + if (threadIdx.x * kNItems + i >= params.seqlen - chunk * kChunkSize) {
244 + thread_data[i] = make_float2(1.f, 0.f);
245 + }
246 + }
247 + }
248 + // Initialize running total
249 + scan_t running_prefix;
250 + // If we use WARP_SCAN then all lane 0 of all warps (not just thread 0) needs to read
251 + running_prefix = chunk == 0 ? x[(r * params.n_chunks) * params.dstate + state_idx] : ( threadIdx.x % 32 == 0 ? smem_running_prefix[state_idx + r * MAX_DSTATE] : make_float2(1.f, 0.f));
252 + // running_prefix = chunk > 0 && threadIdx.x == 0 ? smem_running_prefix[state_idx] : make_float2(1.f, 0.f);
253 + SSMScanPrefixCallbackOp<weight_t> prefix_op(running_prefix);
254 + typename Ktraits::BlockScanT(smem_scan).InclusiveScan(
255 + thread_data, thread_data, SSMScanOp<weight_t>(), prefix_op
256 + );
257 + // There's a syncthreads in the scan op, so we don't need to sync here.
258 + // Unless there's only 1 warp, but then it's the same thread (0) reading and writing.
259 + if (threadIdx.x == 0) {
260 + smem_running_prefix[state_idx] = prefix_op.running_prefix;
261 + x[(r * params.n_chunks + chunk) * params.dstate + state_idx] = prefix_op.running_prefix;
262 + }
263 + #pragma unroll
264 + for (int i = 0; i < kNItems; ++i) {
265 + const weight_t C_val = !kIsVariableC
266 + ? BC_val[r]
267 + : (!kIsVariableB ? BC_val[r] * C_vals[i] : C_vals[i]);
268 + out_vals[r][i] += thread_data[i].y * C_val;
269 + }
270 + }
271 + }
272 +
273 + input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + batch_id * params.out_batch_stride
274 + + dim_id * kNRows * params.out_d_stride + chunk * kChunkSize;
275 + __syncthreads();
276 + #pragma unroll
277 + for (int r = 0; r < kNRows; ++r) {
278 + if constexpr (!kDirectIO) {
279 + if (r > 0) { __syncthreads(); }
280 + }
281 + store_output<Ktraits>(out + r * params.out_d_stride, out_vals[r], smem_store, params.seqlen - chunk * kChunkSize);
282 + }
283 +
284 + if constexpr (kHasZ) {
285 + input_t *z = reinterpret_cast<input_t *>(params.z_ptr) + batch_id * params.z_batch_stride
286 + + dim_id * kNRows * params.z_d_stride + chunk * kChunkSize;
287 + input_t *out_z = reinterpret_cast<input_t *>(params.out_z_ptr) + batch_id * params.out_z_batch_stride
288 + + dim_id * kNRows * params.out_z_d_stride + chunk * kChunkSize;
289 + #pragma unroll
290 + for (int r = 0; r < kNRows; ++r) {
291 + input_t z_vals[kNItems];
292 + __syncthreads();
293 + load_input<Ktraits>(z + r * params.z_d_stride, z_vals, smem_load, params.seqlen - chunk * kChunkSize);
294 + #pragma unroll
295 + for (int i = 0; i < kNItems; ++i) {
296 + float z_val = z_vals[i];
297 + out_vals[r][i] *= z_val / (1 + expf(-z_val));
298 + }
299 + __syncthreads();
300 + store_output<Ktraits>(out_z + r * params.out_z_d_stride, out_vals[r], smem_store, params.seqlen - chunk * kChunkSize);
301 + }
302 + }
303 +
304 + Bvar += kChunkSize * 1;
305 + Cvar += kChunkSize * 1;
306 + }
307 +}
308 +
309 +template<int kNThreads, int kNItems, typename input_t, typename weight_t>
310 +void selective_scan_fwd_launch(SSMParamsBase &params, cudaStream_t stream) {
311 + // Only kNRows == 1 is tested for now, which ofc doesn't differ from previously when we had each block
312 + // processing 1 row.
313 + constexpr int kNRows = 1;
314 + // kIsVariableB, kIsVariableC and kHasZ are all set to True to reduce binary size
315 + constexpr bool kIsVariableB = true;
316 + constexpr bool kIsVariableC = true;
317 + constexpr bool kHasZ = true;
318 + BOOL_SWITCH(params.seqlen % (kNThreads * kNItems) == 0, kIsEvenLen, [&] {
319 + BOOL_SWITCH(params.index_ptr != nullptr , kUseIndex, [&] {
320 + using Ktraits = Selective_Scan_fwd_kernel_traits<kNThreads, kNItems, kNRows, kIsEvenLen, kIsVariableB, kIsVariableC, kHasZ, kUseIndex, input_t, weight_t>;
321 + constexpr int kSmemSize = Ktraits::kSmemSize + kNRows * MAX_DSTATE * sizeof(typename Ktraits::scan_t);
322 + dim3 grid(params.batch, params.dim / kNRows);
323 + auto kernel = &selective_scan_fwd_kernel<Ktraits>;
324 + if (kSmemSize >= 48 * 1024) {
325 + C10_CUDA_CHECK(cudaFuncSetAttribute(
326 + kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
327 + }
328 + kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
329 + C10_CUDA_KERNEL_LAUNCH_CHECK();
330 + });
331 + });
332 +}
333 +
334 +template<typename input_t, typename weight_t>
335 +void selective_scan_fwd_cuda(SSMParamsBase &params, cudaStream_t stream) {
336 +
337 + #ifndef USE_ROCM
338 + if (params.seqlen <= 128) {
339 + selective_scan_fwd_launch<32, 4, input_t, weight_t>(params, stream);
340 + } else if (params.seqlen <= 256) {
341 + selective_scan_fwd_launch<32, 8, input_t, weight_t>(params, stream);
342 + } else if (params.seqlen <= 512) {
343 + selective_scan_fwd_launch<32, 16, input_t, weight_t>(params, stream);
344 + } else if (params.seqlen <= 1024) {
345 + selective_scan_fwd_launch<64, 16, input_t, weight_t>(params, stream);
346 + } else {
347 + selective_scan_fwd_launch<128, 16, input_t, weight_t>(params, stream);
348 + }
349 + #else
350 + if (params.seqlen <= 256) {
351 + selective_scan_fwd_launch<64, 4, input_t, weight_t>(params, stream);
352 + } else if (params.seqlen <= 512) {
353 + selective_scan_fwd_launch<64, 8, input_t, weight_t>(params, stream);
354 + } else if (params.seqlen <= 1024) {
355 + selective_scan_fwd_launch<64, 16, input_t, weight_t>(params, stream);
356 + } else {
357 + selective_scan_fwd_launch<128, 16, input_t, weight_t>(params, stream);
358 + }
359 + #endif
360 +}
361 +
362 +template void selective_scan_fwd_cuda<at::BFloat16, float>(SSMParamsBase &params, cudaStream_t stream);
363 +template void selective_scan_fwd_cuda<at::Half, float>(SSMParamsBase &params, cudaStream_t stream);
364 +template void selective_scan_fwd_cuda<float, float>(SSMParamsBase &params, cudaStream_t stream);
365 +
366 +#define CHECK_SHAPE(x, ...) TORCH_CHECK(x.sizes() == torch::IntArrayRef({__VA_ARGS__}), #x " must have shape (" #__VA_ARGS__ ")")
367 +
368 +#define DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(ITYPE, NAME, ...) \
369 + if (ITYPE == at::ScalarType::Half) { \
370 + using input_t = at::Half; \
371 + using weight_t = float; \
372 + __VA_ARGS__(); \
373 + } else if (ITYPE == at::ScalarType::BFloat16) { \
374 + using input_t = at::BFloat16; \
375 + using weight_t = float; \
376 + __VA_ARGS__(); \
377 + } else if (ITYPE == at::ScalarType::Float) { \
378 + using input_t = float; \
379 + using weight_t = float; \
380 + __VA_ARGS__(); \
381 + } else { \
382 + AT_ERROR(#NAME, " not implemented for input type '", toString(ITYPE), "'"); \
383 + }
384 +
385 +
386 +template<typename input_t, typename weight_t>
387 +void selective_scan_fwd_cuda(SSMParamsBase &params, cudaStream_t stream);
388 +
389 +void set_ssm_params_fwd(SSMParamsBase &params,
390 + // sizes
391 + const size_t batch,
392 + const size_t dim,
393 + const size_t seqlen,
394 + const size_t dstate,
395 + const size_t n_groups,
396 + const size_t n_chunks,
397 + const bool is_variable_B,
398 + const bool is_variable_C,
399 + // device pointers
400 + const torch::Tensor u,
401 + const torch::Tensor delta,
402 + const torch::Tensor A,
403 + const torch::Tensor B,
404 + const torch::Tensor C,
405 + const torch::Tensor out,
406 + const torch::Tensor z,
407 + const torch::Tensor out_z,
408 + void* D_ptr,
409 + void* delta_bias_ptr,
410 + void* x_ptr,
411 + bool has_z,
412 + bool delta_softplus,
413 + void* index_ptr) {
414 +
415 + // Reset the parameters
416 + memset(&params, 0, sizeof(params));
417 +
418 + params.batch = batch;
419 + params.dim = dim;
420 + params.seqlen = seqlen;
421 + params.dstate = dstate;
422 + params.n_groups = n_groups;
423 + params.n_chunks = n_chunks;
424 + params.dim_ngroups_ratio = dim / n_groups;
425 +
426 + params.delta_softplus = delta_softplus;
427 +
428 + params.is_variable_B = is_variable_B;
429 + params.is_variable_C = is_variable_C;
430 +
431 + // Set the pointers and strides.
432 + params.u_ptr = u.data_ptr();
433 + params.delta_ptr = delta.data_ptr();
434 + params.A_ptr = A.data_ptr();
435 + params.B_ptr = B.data_ptr();
436 + params.C_ptr = C.data_ptr();
437 + params.D_ptr = D_ptr;
438 + params.delta_bias_ptr = delta_bias_ptr;
439 + params.out_ptr = out.data_ptr();
440 + params.x_ptr = x_ptr;
441 + params.z_ptr = has_z ? z.data_ptr() : nullptr;
442 + params.out_z_ptr = has_z ? out_z.data_ptr() : nullptr;
443 +
444 + params.index_ptr = index_ptr;
445 +
446 + // All stride are in elements, not bytes.
447 + params.A_d_stride = A.stride(0);
448 + params.A_dstate_stride = A.stride(1);
449 + if (!is_variable_B) {
450 + params.B_d_stride = B.stride(0);
451 + } else {
452 + params.B_batch_stride = B.stride(0);
453 + params.B_group_stride = B.stride(1);
454 + }
455 + params.B_dstate_stride = !is_variable_B ? B.stride(1) : B.stride(2);
456 + if (!is_variable_C) {
457 + params.C_d_stride = C.stride(0);
458 + } else {
459 + params.C_batch_stride = C.stride(0);
460 + params.C_group_stride = C.stride(1);
461 + }
462 + params.C_dstate_stride = !is_variable_C ? C.stride(1) : C.stride(2);
463 + params.u_batch_stride = u.stride(0);
464 + params.u_d_stride = u.stride(1);
465 + params.delta_batch_stride = delta.stride(0);
466 + params.delta_d_stride = delta.stride(1);
467 + if (has_z) {
468 + params.z_batch_stride = z.stride(0);
469 + params.z_d_stride = z.stride(1);
470 + params.out_z_batch_stride = out_z.stride(0);
471 + params.out_z_d_stride = out_z.stride(1);
472 + }
473 + params.out_batch_stride = out.stride(0);
474 + params.out_d_stride = out.stride(1);
475 +}
476 +
477 +std::vector<torch::Tensor>
478 +selective_scan_fwd(const torch::Tensor &u, const torch::Tensor &delta,
479 + const torch::Tensor &A, const torch::Tensor &B, const torch::Tensor &C,
480 + const c10::optional<torch::Tensor> &D_,
481 + const c10::optional<torch::Tensor> &z_,
482 + const c10::optional<torch::Tensor> &delta_bias_,
483 + bool delta_softplus,
484 + const c10::optional<torch::Tensor> &index_,
485 + const c10::optional<torch::Tensor> &x) {
486 + auto input_type = u.scalar_type();
487 + auto weight_type = A.scalar_type();
488 + TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16);
489 + TORCH_CHECK(weight_type == at::ScalarType::Float);
490 +
491 + const bool is_variable_B = B.dim() >= 3;
492 + const bool is_variable_C = C.dim() >= 3;
493 +
494 + TORCH_CHECK(delta.scalar_type() == input_type);
495 + TORCH_CHECK(B.scalar_type() == (!is_variable_B ? weight_type : input_type));
496 + TORCH_CHECK(C.scalar_type() == (!is_variable_C ? weight_type : input_type));
497 +
498 + TORCH_CHECK(u.is_cuda());
499 + TORCH_CHECK(delta.is_cuda());
500 + TORCH_CHECK(A.is_cuda());
501 + TORCH_CHECK(B.is_cuda());
502 + TORCH_CHECK(C.is_cuda());
503 +
504 + TORCH_CHECK(u.stride(-1) == 1 || u.size(-1) == 1);
505 + TORCH_CHECK(delta.stride(-1) == 1 || delta.size(-1) == 1);
506 +
507 + const auto sizes = u.sizes();
508 + const int batch_size = sizes[0];
509 + const int dim = sizes[1];
510 + const int seqlen = sizes[2];
511 + const int dstate = A.size(1);
512 + const int n_groups = is_variable_B ? B.size(1) : 1;
513 +
514 + TORCH_CHECK(dstate <= 256, "selective_scan only supports state dimension <= 256");
515 +
516 + CHECK_SHAPE(u, batch_size, dim, seqlen);
517 + CHECK_SHAPE(delta, batch_size, dim, seqlen);
518 + CHECK_SHAPE(A, dim, dstate);
519 + TORCH_CHECK(is_variable_B, "is_variable_B = False is disabled in favor of reduced binary size")
520 + CHECK_SHAPE(B, batch_size, n_groups, dstate, seqlen );
521 + TORCH_CHECK(B.stride(-1) == 1 || B.size(-1) == 1);
522 +
523 + TORCH_CHECK(is_variable_C, "is_variable_C = False is disabled in favor of reduced binary size")
524 + CHECK_SHAPE(C, batch_size, n_groups, dstate, seqlen);
525 + TORCH_CHECK(C.stride(-1) == 1 || C.size(-1) == 1);
526 +
527 + if (D_.has_value()) {
528 + auto D = D_.value();
529 + TORCH_CHECK(D.scalar_type() == at::ScalarType::Float);
530 + TORCH_CHECK(D.is_cuda());
531 + TORCH_CHECK(D.stride(-1) == 1 || D.size(-1) == 1);
532 + CHECK_SHAPE(D, dim);
533 + }
534 +
535 + if (delta_bias_.has_value()) {
536 + auto delta_bias = delta_bias_.value();
537 + TORCH_CHECK(delta_bias.scalar_type() == at::ScalarType::Float);
538 + TORCH_CHECK(delta_bias.is_cuda());
539 + TORCH_CHECK(delta_bias.stride(-1) == 1 || delta_bias.size(-1) == 1);
540 + CHECK_SHAPE(delta_bias, dim);
541 + }
542 + if (index_.has_value()) {
543 + auto index = index_.value();
544 + TORCH_CHECK(index.scalar_type() == at::ScalarType::Int);
545 + TORCH_CHECK(index.is_cuda());
546 + CHECK_SHAPE(index, batch_size, seqlen);
547 + }
548 +
549 + at::Tensor z, out_z;
550 + const bool has_z = z_.has_value();
551 + TORCH_CHECK(has_z, "has_z = False is disabled in favor of reduced binary size")
552 + z = z_.value();
553 + TORCH_CHECK(z.scalar_type() == input_type);
554 + TORCH_CHECK(z.is_cuda());
555 + TORCH_CHECK(z.stride(-1) == 1 || z.size(-1) == 1);
556 + CHECK_SHAPE(z, batch_size, dim, seqlen);
557 + out_z = torch::empty_like(z);
558 +
559 + const int n_chunks = (seqlen + 2048 - 1) / 2048;
560 + // const int n_chunks = (seqlen + 1024 - 1) / 1024;
561 + // at::Tensor out = torch::empty_like(u);
562 + // Right now u has BHL layout and delta has HBL layout, and we want out to have HBL layout
563 + at::Tensor out = torch::empty_like(delta);
564 + if (x.has_value()){
565 + auto _x = x.value();
566 + TORCH_CHECK(_x.scalar_type() == weight_type);
567 + TORCH_CHECK(_x.is_cuda());
568 + TORCH_CHECK(_x.stride(-1) == 1);
569 + CHECK_SHAPE(_x, batch_size, dim, n_chunks, dstate * 2);
570 + }
571 +
572 + SSMParamsBase params;
573 + set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, n_chunks, is_variable_B, is_variable_C,
574 + u, delta, A, B, C, out, z, out_z,
575 + D_.has_value() ? D_.value().data_ptr() : nullptr,
576 + delta_bias_.has_value() ? delta_bias_.value().data_ptr() : nullptr,
577 + x.value().data_ptr(),
578 + has_z,
579 + delta_softplus,
580 + index_.has_value() ? index_.value().data_ptr() : nullptr);
581 +
582 + // Otherwise the kernel will be launched from cuda:0 device
583 + // Cast to char to avoid compiler warning about narrowing
584 + at::cuda::CUDAGuard device_guard{(char)u.get_device()};
585 + auto stream = at::cuda::getCurrentCUDAStream().stream();
586 + DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(u.scalar_type(), "selective_scan_fwd", [&] {
587 + selective_scan_fwd_cuda<input_t, weight_t>(params, stream);
588 + });
589 + std::vector<at::Tensor> result = {out, x.value()};
590 + if (has_z) { result.push_back(out_z); }
591 + return result;
592 +}
593 +
csrc/mamba/mamba_ssm/static_switch.h new
+28
@@ -0,0 +1,28 @@
1 +// Inspired by
2 +// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h
3 +// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h
4 +
5 +// clang-format off
6 +// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/static_switch.h
7 +#pragma once
8 +
9 +/// @param COND - a boolean expression to switch by
10 +/// @param CONST_NAME - a name given for the constexpr bool variable.
11 +/// @param ... - code to execute for true and false
12 +///
13 +/// Usage:
14 +/// ```
15 +/// BOOL_SWITCH(flag, BoolConst, [&] {
16 +/// some_function<BoolConst>(...);
17 +/// });
18 +/// ```
19 +#define BOOL_SWITCH(COND, CONST_NAME, ...) \
20 + [&] { \
21 + if (COND) { \
22 + constexpr bool CONST_NAME = true; \
23 + return __VA_ARGS__(); \
24 + } else { \
25 + constexpr bool CONST_NAME = false; \
26 + return __VA_ARGS__(); \
27 + } \
28 + }()
csrc/ops.h
+22
@@ -195,6 +195,28 @@ void moe_align_block_size(torch::Tensor topk_ids, int64_t num_experts,
195 torch::Tensor experts_ids,
196 torch::Tensor num_tokens_post_pad);
197
198 +std::vector<torch::Tensor> selective_scan_fwd(
199 + const torch::Tensor& u, const torch::Tensor& delta, const torch::Tensor& A,
200 + const torch::Tensor& B, const torch::Tensor& C,
201 + const c10::optional<torch::Tensor>& D_,
202 + const c10::optional<torch::Tensor>& z_,
203 + const c10::optional<torch::Tensor>& delta_bias_, bool delta_softplus,
204 + const c10::optional<torch::Tensor>& index_,
205 + const c10::optional<torch::Tensor>& x);
206 +
207 +at::Tensor causal_conv1d_update(const at::Tensor& x,
208 + const at::Tensor& conv_state,
209 + const at::Tensor& weight,
210 + const c10::optional<at::Tensor>& bias_,
211 + bool silu_activation);
212 +
213 +at::Tensor causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
214 + const c10::optional<at::Tensor>& bias_,
215 + const c10::optional<at::Tensor>& seq_idx_,
216 + const c10::optional<at::Tensor>& initial_states_,
217 + const c10::optional<at::Tensor>& final_states_out_,
218 + bool silu_activation);
219 +
220 #ifndef USE_ROCM
221 using fptr_t = int64_t;
222 fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data,
csrc/torch_bindings.cpp
+25
@@ -202,6 +202,31 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
202 ops.def("cutlass_scaled_mm_supports_fp8", &cutlass_scaled_mm_supports_fp8);
203 ops.impl("cutlass_scaled_mm_supports_fp8", torch::kCUDA,
204 &cutlass_scaled_mm_supports_fp8);
205 + // Mamba selective scan kernel
206 + ops.def(
207 + "selective_scan_fwd(Tensor! u, Tensor! delta,"
208 + "Tensor! A, Tensor! B, Tensor! C,"
209 + "Tensor? D_, Tensor? z_, Tensor? delta_bias_,"
210 + "bool delta_softplus,"
211 + "Tensor? index_, Tensor? x) -> Tensor[]");
212 + ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
213 +
214 + ops.def(
215 + "causal_conv1d_update(Tensor! x,"
216 + "Tensor! conv_state,"
217 + "Tensor! weight,"
218 + "Tensor? bias_,"
219 + "bool silu_activation) -> Tensor");
220 + ops.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update);
221 +
222 + ops.def(
223 + "causal_conv1d_fwd(Tensor! x, Tensor! weight,"
224 + "Tensor? bias_,"
225 + "Tensor? seq_idx_,"
226 + "Tensor? initial_states_,"
227 + "Tensor? final_states_out_,"
228 + "bool silu_activation) -> Tensor");
229 + ops.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd);
230 #endif
231
232 // Quantized GEMM for GPTQ.
requirements-mamba.txt deleted
-3
@@ -1,3 +0,0 @@
1 -# Mamba dependencies
2 -mamba-ssm>=1.2.2
3 -causal-conv1d>=1.2.0
requirements-test.txt
+1 -1
@@ -11,7 +11,7 @@ pytest-shard
11
12 # testing utils
13 awscli
14 -einops # required for MPT and qwen-vl
14 +einops # required for MPT, qwen-vl and Mamba
15 httpx
16 peft
17 requests
tests/kernels/test_causal_conv1d.py new
+205
@@ -0,0 +1,205 @@
1 +from typing import Optional
2 +
3 +import pytest
4 +import torch
5 +import torch.nn.functional as F
6 +from einops import rearrange
7 +
8 +from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
9 + causal_conv1d_fn, causal_conv1d_update)
10 +
11 +
12 +def causal_conv1d_ref(
13 + x: torch.Tensor,
14 + weight: torch.Tensor,
15 + bias: Optional[torch.Tensor] = None,
16 + initial_states: Optional[torch.Tensor] = None,
17 + return_final_states: bool = False,
18 + final_states_out: Optional[torch.Tensor] = None,
19 + activation: Optional[str] = "silu",
20 +):
21 + """
22 + x: (batch, dim, seqlen)
23 + weight: (dim, width)
24 + bias: (dim,)
25 + initial_states: (batch, dim, width - 1)
26 + final_states_out: (batch, dim, width - 1)
27 +
28 + out: (batch, dim, seqlen)
29 + """
30 + if activation not in [None, "silu", "swish"]:
31 + raise NotImplementedError("activation must be None, silu, or swish")
32 + dtype_in = x.dtype
33 + x = x.to(weight.dtype)
34 + seqlen = x.shape[-1]
35 + dim, width = weight.shape
36 + if initial_states is None:
37 + out = F.conv1d(x,
38 + weight.unsqueeze(1),
39 + bias,
40 + padding=width - 1,
41 + groups=dim)
42 + else:
43 + x = torch.cat([initial_states, x], dim=-1)
44 + out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim)
45 + out = out[..., :seqlen]
46 + if return_final_states:
47 + final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to(
48 + dtype_in) # (batch, dim, width - 1)
49 + if final_states_out is not None:
50 + final_states_out.copy_(final_states)
51 + else:
52 + final_states_out = final_states
53 + out = (out if activation is None else F.silu(out)).to(dtype=dtype_in)
54 + return (out, None) if not return_final_states else (out, final_states_out)
55 +
56 +
57 +def causal_conv1d_update_ref(x: torch.Tensor,
58 + conv_state: torch.Tensor,
59 + weight: torch.Tensor,
60 + bias: Optional[torch.Tensor] = None,
61 + activation: Optional[str] = None):
62 + """
63 + x: (batch, dim)
64 + conv_state: (batch, dim, width)
65 + weight: (dim, width)
66 + bias: (dim,)
67 +
68 + out: (batch, dim)
69 + """
70 + if activation not in [None, "silu", "swish"]:
71 + raise NotImplementedError("activation must be None, silu, or swish")
72 + dtype_in = x.dtype
73 + batch, dim = x.shape
74 + width = weight.shape[1]
75 + assert conv_state.shape == (batch, dim, width)
76 + assert weight.shape == (dim, width)
77 + conv_state.copy_(torch.roll(conv_state, shifts=-1,
78 + dims=-1)) # Update state (B D W)
79 + conv_state[:, :, -1] = x
80 + out = torch.sum(conv_state * weight, dim=-1) # (B D)
81 + if bias is not None:
82 + out += bias
83 + return (out if activation is None else F.silu(out)).to(dtype=dtype_in)
84 +
85 +
86 +@pytest.mark.parametrize("return_final_states", [False, True])
87 +@pytest.mark.parametrize("has_initial_states", [False, True])
88 +@pytest.mark.parametrize("channel_last", [False, True])
89 +@pytest.mark.parametrize("itype", [torch.bfloat16])
90 +@pytest.mark.parametrize("silu_activation", [False, True])
91 +@pytest.mark.parametrize("has_bias", [False, True])
92 +@pytest.mark.parametrize("width", [4])
93 +@pytest.mark.parametrize("seqlen", [128, 512, 4096])
94 +@pytest.mark.parametrize('dim', [64, 4096 + 32])
95 +@pytest.mark.parametrize('batch', [1, 2])
96 +def test_causal_conv1d(batch, dim, seqlen, width, has_bias, silu_activation,
97 + itype, channel_last, has_initial_states,
98 + return_final_states):
99 + if not channel_last and (has_initial_states or return_final_states):
100 + pytest.skip(
101 + "Only channel_last support initial_states or return_final_states")
102 + device = "cuda"
103 + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
104 + if itype == torch.bfloat16:
105 + rtol, atol = 1e-2, 5e-2
106 + # set seed
107 + torch.random.manual_seed(0)
108 + if not channel_last:
109 + x = torch.randn(batch,
110 + 4096 + dim + 64,
111 + seqlen,
112 + device=device,
113 + dtype=itype)[:, 4096:4096 + dim, :]
114 + else:
115 + x = rearrange(
116 + torch.randn(batch,
117 + seqlen,
118 + 4096 + dim + 64,
119 + device=device,
120 + dtype=itype)[:, :, 4096:4096 + dim], "b s d -> b d s")
121 + weight = torch.randn(dim, width, device=device, dtype=itype)
122 + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
123 + if has_initial_states:
124 + initial_states = torch.randn(batch,
125 + width - 1,
126 + dim,
127 + device=device,
128 + dtype=itype).transpose(1, 2)
129 + else:
130 + initial_states = None
131 + x_ref = x.detach().clone()
132 + weight_ref = weight.detach().clone()
133 + bias_ref = bias.detach().clone() if bias is not None else None
134 + initial_states_ref = initial_states.detach().clone(
135 + ) if initial_states is not None else None
136 + activation = None if not silu_activation else "silu"
137 + out, final_states = causal_conv1d_fn(
138 + x,
139 + weight,
140 + bias,
141 + initial_states=initial_states,
142 + return_final_states=return_final_states,
143 + activation=activation)
144 + out_ref, final_states_ref = causal_conv1d_ref(
145 + x_ref,
146 + weight_ref,
147 + bias_ref,
148 + initial_states=initial_states_ref,
149 + return_final_states=return_final_states,
150 + activation=activation)
151 + if return_final_states:
152 + assert final_states is not None and final_states_ref is not None
153 + assert torch.allclose(final_states,
154 + final_states_ref,
155 + rtol=rtol,
156 + atol=atol)
157 +
158 + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
159 +
160 + if return_final_states:
161 + out += F.sigmoid(final_states).sum(dim=-1, keepdim=True)
162 + out_ref += F.sigmoid(final_states_ref).sum(dim=-1, keepdim=True)
163 +
164 +
165 +@pytest.mark.parametrize("itype", [torch.bfloat16])
166 +@pytest.mark.parametrize("silu_activation", [False, True])
167 +@pytest.mark.parametrize("has_bias", [False, True])
168 +@pytest.mark.parametrize("width", [2, 3, 4])
169 +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
170 +@pytest.mark.parametrize("batch", [1, 2])
171 +def test_causal_conv1d_update(batch, dim, width, has_bias, silu_activation,
172 + itype):
173 + device = "cuda"
174 + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3)
175 + if itype == torch.bfloat16:
176 + rtol, atol = 1e-2, 5e-2
177 + # set seed
178 + torch.random.manual_seed(0)
179 + batch = 2
180 + x = torch.randn(batch, dim, device=device, dtype=itype)
181 + conv_state = torch.randn(batch, dim, width, device=device, dtype=itype)
182 + weight = torch.randn(dim,
183 + width,
184 + device=device,
185 + dtype=itype,
186 + requires_grad=True)
187 + if has_bias:
188 + bias = torch.randn(dim, device=device, dtype=itype, requires_grad=True)
189 + else:
190 + bias = None
191 + conv_state_ref = conv_state.detach().clone()
192 + activation = None if not silu_activation else "silu"
193 + out = causal_conv1d_update(x,
194 + conv_state,
195 + weight,
196 + bias,
197 + activation=activation)
198 + out_ref = causal_conv1d_update_ref(x,
199 + conv_state_ref,
200 + weight,
201 + bias,
202 + activation=activation)
203 +
204 + assert torch.equal(conv_state, conv_state_ref)
205 + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
tests/kernels/test_mamba_ssm.py new
+324
@@ -0,0 +1,324 @@
1 +import pytest
2 +import torch
3 +import torch.nn.functional as F
4 +from einops import rearrange, repeat
5 +
6 +from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
7 + selective_scan_fn, selective_state_update)
8 +
9 +
10 +def selective_state_update_ref(state,
11 + x,
12 + dt,
13 + A,
14 + B,
15 + C,
16 + D=None,
17 + z=None,
18 + dt_bias=None,
19 + dt_softplus=False):
20 + """
21 + Argument:
22 + state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
23 + x: (batch, dim) or (batch, nheads, dim)
24 + dt: (batch, dim) or (batch, nheads, dim)
25 + A: (dim, dstate) or (nheads, dim, dstate)
26 + B: (batch, dstate) or (batch, ngroups, dstate)
27 + C: (batch, dstate) or (batch, ngroups, dstate)
28 + D: (dim,) or (nheads, dim)
29 + z: (batch, dim) or (batch, nheads, dim)
30 + dt_bias: (dim,) or (nheads, dim)
31 + Return:
32 + out: (batch, dim) or (batch, nheads, dim)
33 + """
34 + has_heads = state.dim() > 3
35 + if state.dim() == 3:
36 + state = state.unsqueeze(1)
37 + if x.dim() == 2:
38 + x = x.unsqueeze(1)
39 + if dt.dim() == 2:
40 + dt = dt.unsqueeze(1)
41 + if A.dim() == 2:
42 + A = A.unsqueeze(0)
43 + if B.dim() == 2:
44 + B = B.unsqueeze(1)
45 + if C.dim() == 2:
46 + C = C.unsqueeze(1)
47 + if D is not None and D.dim() == 1:
48 + D = D.unsqueeze(0)
49 + if z is not None and z.dim() == 2:
50 + z = z.unsqueeze(1)
51 + if dt_bias is not None and dt_bias.dim() == 1:
52 + dt_bias = dt_bias.unsqueeze(0)
53 + batch, nheads, dim, dstate = state.shape
54 + assert x.shape == (batch, nheads, dim)
55 + assert dt.shape == x.shape
56 + assert A.shape == (nheads, dim, dstate)
57 + ngroups = B.shape[1]
58 + assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
59 + assert B.shape == (batch, ngroups, dstate)
60 + assert C.shape == B.shape
61 + if D is not None:
62 + assert D.shape == (nheads, dim)
63 + if z is not None:
64 + assert z.shape == x.shape
65 + if dt_bias is not None:
66 + assert dt_bias.shape == (nheads, dim)
67 + dt = dt + dt_bias
68 + dt = F.softplus(dt) if dt_softplus else dt
69 + dA = torch.exp(rearrange(dt, "b h d -> b h d 1") *
70 + A) # (batch, nheads, dim, dstate)
71 + B = repeat(B, "b g n -> b (g h) n",
72 + h=nheads // ngroups) # (batch, nheads, dstate)
73 + C = repeat(C, "b g n -> b (g h) n",
74 + h=nheads // ngroups) # (batch, nheads, dstate)
75 + dB = rearrange(dt, "b h d -> b h d 1") * rearrange(
76 + B, "b h n -> b h 1 n") # (batch, nheads, dim, dstate)
77 + state.copy_(state * dA +
78 + dB * rearrange(x, "b h d -> b h d 1")) # (batch, dim, dstate
79 + out = torch.einsum("bhdn,bhn->bhd", state.to(C.dtype), C)
80 + if D is not None:
81 + out += (x * D).to(out.dtype)
82 + out = (out if z is None else out * F.silu(z)).to(x.dtype)
83 + if not has_heads:
84 + out = out.squeeze(1)
85 + return out
86 +
87 +
88 +def selective_scan_ref(u,
89 + delta,
90 + A,
91 + B,
92 + C,
93 + D=None,
94 + z=None,
95 + delta_bias=None,
96 + delta_softplus=False,
97 + return_last_state=False,
98 + position_indices=None,
99 + prev_state=None):
100 + """
101 + u: r(B D L)
102 + delta: r(B D L)
103 + A: c(D N) or r(D N)
104 + B: c(D N) or r(B N L) or r(B N 2L) or r(B G N L) or (B G N L)
105 + C: c(D N) or r(B N L) or r(B N 2L) or r(B G N L) or (B G N L)
106 + D: r(D)
107 + z: r(B D L)
108 + delta_bias: r(D), fp32
109 + prev_state: r(B D N), fp32
110 +
111 + out: r(B D L)
112 + last_state (optional): r(B D dstate) or c(B D dstate)
113 + """
114 + dtype_in = u.dtype
115 + u = u.float()
116 + delta = delta.float()
117 + if delta_bias is not None:
118 + delta = delta + delta_bias[..., None].float()
119 + if delta_softplus:
120 + delta = F.softplus(delta)
121 + batch, dim, dstate = u.shape[0], A.shape[0], A.shape[1]
122 + is_variable_B = B.dim() >= 3
123 + is_variable_C = C.dim() >= 3
124 + B = B.float()
125 + C = C.float()
126 + x = A.new_zeros((batch, dim, dstate)) if prev_state is None else prev_state
127 + ys = []
128 + deltaA = torch.exp(torch.einsum('bdl,dn->bdln', delta, A))
129 + if not is_variable_B:
130 + deltaB_u = torch.einsum('bdl,dn,bdl->bdln', delta, B, u)
131 + else:
132 + if B.dim() == 3:
133 + deltaB_u = torch.einsum('bdl,bnl,bdl->bdln', delta, B, u)
134 + else:
135 + B = repeat(B, "B G N L -> B (G H) N L", H=dim // B.shape[1])
136 + deltaB_u = torch.einsum('bdl,bdnl,bdl->bdln', delta, B, u)
137 + if is_variable_C and C.dim() == 4:
138 + C = repeat(C, "B G N L -> B (G H) N L", H=dim // C.shape[1])
139 + last_state = None
140 + for i in range(u.shape[2]):
141 + if position_indices is not None and position_indices[0, i] == 0:
142 + x = deltaB_u[:, :, i]
143 + else:
144 + x = deltaA[:, :, i] * x + deltaB_u[:, :, i]
145 + if not is_variable_C:
146 + y = torch.einsum('bdn,dn->bd', x, C)
147 + else:
148 + if C.dim() == 3:
149 + y = torch.einsum('bdn,bn->bd', x, C[:, :, i])
150 + else:
151 + y = torch.einsum('bdn,bdn->bd', x, C[:, :, :, i])
152 + if i == u.shape[2] - 1:
153 + last_state = x
154 + ys.append(y)
155 + y = torch.stack(ys, dim=2) # (batch dim L)
156 + out = y if D is None else y + u * rearrange(D, "d -> d 1")
157 + if z is not None:
158 + out = out * F.silu(z)
159 + out = out.to(dtype=dtype_in)
160 + return out if not return_last_state else (out, last_state)
161 +
162 +
163 +@pytest.mark.parametrize('wtype', [torch.float32])
164 +@pytest.mark.parametrize('itype', [torch.float32])
165 +@pytest.mark.parametrize('seqlen', [128, 256, 512, 1024, 2048, 4096])
166 +@pytest.mark.parametrize("return_last_state", [True])
167 +@pytest.mark.parametrize('has_delta_bias', [True])
168 +@pytest.mark.parametrize('delta_softplus', [True])
169 +@pytest.mark.parametrize('has_z', [True])
170 +@pytest.mark.parametrize('has_D', [True])
171 +@pytest.mark.parametrize("varBC_groups", [1, 2])
172 +@pytest.mark.parametrize("is_variable_C", [True])
173 +@pytest.mark.parametrize("is_variable_B", [True])
174 +@pytest.mark.parametrize("scan_chunks", [1, 2, 3])
175 +def test_selective_scan(is_variable_B, is_variable_C, varBC_groups, has_D,
176 + has_z, has_delta_bias, delta_softplus,
177 + return_last_state, seqlen, itype, wtype, scan_chunks):
178 + if varBC_groups > 1 and (not is_variable_B or not is_variable_C):
179 + pytest.skip() # This config is not applicable
180 + device = 'cuda'
181 + rtol, atol = (6e-4, 2e-3) if itype == torch.float32 else (3e-3, 5e-3)
182 + if itype == torch.bfloat16:
183 + rtol, atol = 3e-2, 5e-2
184 + rtolw, atolw = (1e-3, 1e-3)
185 + if has_z: # If we have z, the errors on the weights seem higher
186 + rtolw = max(rtolw, rtol)
187 + atolw = max(atolw, atol)
188 + # set seed
189 + torch.random.manual_seed(0)
190 + batch_size = 2
191 + dim = 4
192 + dstate = 8
193 + A = (-0.5 * torch.rand(dim, dstate, device=device, dtype=wtype))
194 + if not is_variable_B:
195 + B_shape = [dim, dstate]
196 + elif varBC_groups == 1:
197 + B_shape = [batch_size, dstate, seqlen]
198 + else:
199 + B_shape = [batch_size, varBC_groups, dstate, seqlen]
200 + B = torch.randn(B_shape,
201 + device=device,
202 + dtype=wtype if not is_variable_B else itype)
203 + if not is_variable_C:
204 + C_shape = [dim, dstate]
205 + elif varBC_groups == 1:
206 + C_shape = [batch_size, dstate, seqlen]
207 + else:
208 + C_shape = [batch_size, varBC_groups, dstate, seqlen]
209 + C = torch.randn(C_shape,
210 + device=device,
211 + dtype=wtype if not is_variable_C else itype)
212 + D = torch.randn(dim, device=device, dtype=torch.float32) if has_D else None
213 + z = torch.randn(batch_size, dim, seqlen, device=device,
214 + dtype=itype) if has_z else None
215 + delta_bias = (0.5 * torch.rand(dim, device=device, dtype=torch.float32)
216 + ) if has_delta_bias else None
217 + u = torch.randn(batch_size, dim, seqlen, device=device, dtype=itype)
218 + delta = (0.5 *
219 + torch.rand(batch_size, dim, seqlen, device=device, dtype=itype))
220 + state = None
221 + state_ref = None
222 + out = None
223 + out_ref = None
224 + outs = []
225 + for c in range(scan_chunks):
226 + chunked_prompt_len = seqlen // scan_chunks
227 + chunk_start = chunked_prompt_len * c
228 + chunk_end = chunked_prompt_len * (c + 1)
229 + if c == scan_chunks - 1:
230 + chunk_end = seqlen
231 + _B = B
232 + if is_variable_B:
233 + _B = B[..., chunk_start:chunk_end]
234 + _C = C
235 + if is_variable_B:
236 + _C = C[..., chunk_start:chunk_end]
237 + _z = z
238 + if has_z:
239 + assert z is not None
240 + _z = z[..., chunk_start:chunk_end]
241 + out, *rest = selective_scan_fn(u[..., chunk_start:chunk_end],
242 + delta[..., chunk_start:chunk_end],
243 + A,
244 + _B,
245 + _C,
246 + D,
247 + z=_z,
248 + delta_bias=delta_bias,
249 + delta_softplus=delta_softplus,
250 + return_last_state=return_last_state,
251 + prev_state=state if c > 0 else None)
252 + outs.append(out)
253 + if return_last_state:
254 + state = rest[0]
255 + if len(outs) > 1:
256 + out = torch.cat(outs, dim=-1)
257 + out_ref, *rest = selective_scan_ref(u,
258 + delta,
259 + A,
260 + B,
261 + C,
262 + D,
263 + z=z,
264 + delta_bias=delta_bias,
265 + delta_softplus=delta_softplus,
266 + return_last_state=return_last_state)
267 + if return_last_state:
268 + state_ref = rest[0]
269 +
270 + assert out is not None and out_ref is not None
271 + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
272 + if return_last_state:
273 + assert state is not None and state_ref is not None
274 + assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
275 +
276 +
277 +@pytest.mark.parametrize("itype",
278 + [torch.float32, torch.float16, torch.bfloat16])
279 +@pytest.mark.parametrize("has_z", [False, True])
280 +@pytest.mark.parametrize("dstate", [16, 32, 64])
281 +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096])
282 +def test_selective_state_update(dim, dstate, has_z, itype):
283 + device = "cuda"
284 + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
285 + if itype == torch.bfloat16:
286 + rtol, atol = 1e-2, 5e-2
287 + if torch.version.hip:
288 + atol *= 2
289 + # set seed
290 + torch.random.manual_seed(0)
291 + batch_size = 1
292 + state = torch.randn(batch_size, dim, dstate, dtype=itype, device=device)
293 + x = torch.randn(batch_size, dim, device=device, dtype=itype)
294 + dt = torch.randn(batch_size, dim, device=device, dtype=itype)
295 + dt_bias = torch.rand(dim, device=device) - 4.0
296 + A = -torch.rand(dim, dstate, device=device) - 1.0
297 + B = torch.randn(batch_size, dstate, device=device)
298 + C = torch.randn(batch_size, dstate, device=device)
299 + D = torch.randn(dim, device=device)
300 + z = torch.randn_like(x) if has_z else None
301 + state_ref = state.detach().clone()
302 + out = selective_state_update(state,
303 + x,
304 + dt,
305 + A,
306 + B,
307 + C,
308 + D=D,
309 + z=z,
310 + dt_bias=dt_bias,
311 + dt_softplus=True)
312 + out_ref = selective_state_update_ref(state_ref,
313 + x,
314 + dt,
315 + A,
316 + B,
317 + C,
318 + D=D,
319 + z=z,
320 + dt_bias=dt_bias,
321 + dt_softplus=True)
322 +
323 + assert torch.allclose(state, state_ref, rtol=rtol, atol=atol)
324 + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol)
vllm/_custom_ops.py
+30
@@ -500,6 +500,36 @@ def ggml_mul_mat_a8(
500 return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
501
502
503 +# mamba
504 +def causal_conv1d_fwd(x: torch.Tensor, weight: torch.Tensor,
505 + bias_: Optional[torch.Tensor],
506 + seq_idx_: Optional[torch.Tensor],
507 + initial_states_: Optional[torch.Tensor],
508 + final_states_out_: Optional[torch.Tensor],
509 + silu_activation: bool) -> torch.Tensor:
510 + return torch.ops._C.causal_conv1d_fwd(x, weight, bias_, seq_idx_,
511 + initial_states_, final_states_out_,
512 + silu_activation)
513 +
514 +
515 +def causal_conv1d_update(x: torch.Tensor, conv_state: torch.Tensor,
516 + weight: torch.Tensor, bias_: Optional[torch.Tensor],
517 + silu_activation: bool) -> torch.Tensor:
518 + return torch.ops._C.causal_conv1d_update(x, conv_state, weight, bias_,
519 + silu_activation)
520 +
521 +
522 +def selective_scan_fwd(u: torch.Tensor, delta: torch.Tensor, A: torch.Tensor,
523 + B: torch.Tensor, C: torch.Tensor,
524 + D_: Optional[torch.Tensor], z_: Optional[torch.Tensor],
525 + delta_bias_: Optional[torch.Tensor],
526 + delta_softplus: bool, index_: Optional[torch.Tensor],
527 + x: Optional[torch.Tensor]) -> List[torch.Tensor]:
528 + return torch.ops._C.selective_scan_fwd(u, delta, A, B, C, D_, z_,
529 + delta_bias_, delta_softplus, index_,
530 + x)
531 +
532 +
533 # moe
534 def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
535 block_size: int, sorted_token_ids: torch.Tensor,
vllm/model_executor/layers/mamba/__init__.py
vllm/model_executor/layers/mamba/ops/__init__.py
vllm/model_executor/layers/mamba/ops/causal_conv1d.py new
+86
@@ -0,0 +1,86 @@
1 +# Copyright (c) 2024, Tri Dao.
2 +
3 +from typing import Optional
4 +
5 +import torch
6 +
7 +from vllm import _custom_ops as ops
8 +
9 +
10 +def causal_conv1d_fn(
11 + x: torch.Tensor,
12 + weight: torch.Tensor,
13 + bias: Optional[torch.Tensor] = None,
14 + seq_idx: Optional[torch.Tensor] = None,
15 + initial_states: Optional[torch.Tensor] = None,
16 + return_final_states: bool = False,
17 + final_states_out=None,
18 + activation: str = "silu",
19 +):
20 + """
21 + x: (batch, dim, seqlen)
22 + weight: (dim, width)
23 + bias: (dim,)
24 + seq_idx: (batch, seqlen)
25 + initial_states: (batch, dim, width - 1)
26 + final_states_out: (batch, dim, width - 1), to be written to
27 + activation: either None or "silu" or "swish"
28 +
29 + out: (batch, dim, seqlen)
30 + """
31 + if activation not in [None, "silu", "swish"]:
32 + raise NotImplementedError("activation must be None, silu, or swish")
33 + if x.stride(2) != 1 and x.stride(1) != 1:
34 + x = x.contiguous()
35 + bias = bias.contiguous() if bias is not None else None
36 + if seq_idx is not None:
37 + assert (initial_states is
38 + None), "initial_states must be None if seq_idx is not None"
39 + assert (not return_final_states
40 + ), "If seq_idx is not None, we don't return final_states_out"
41 + seq_idx = seq_idx.contiguous() if seq_idx is not None else None
42 + if initial_states is not None and (initial_states.stride(2) != 1
43 + and initial_states.stride(1) != 1):
44 + initial_states = initial_states.contiguous()
45 + if return_final_states:
46 + assert (
47 + x.stride(1) == 1
48 + ), "Only channel-last layout support returning final_states_out"
49 + if final_states_out is not None:
50 + assert (final_states_out.stride(2) == 1
51 + or final_states_out.stride(1) == 1)
52 + else:
53 + batch, dim, seqlen = x.shape
54 + width = weight.shape[1]
55 + final_states_out = torch.empty(batch,
56 + width - 1,
57 + dim,
58 + device=x.device,
59 + dtype=x.dtype).transpose(1, 2)
60 + else:
61 + final_states_out = None
62 +
63 + out = ops.causal_conv1d_fwd(x, weight, bias, seq_idx, initial_states,
64 + final_states_out, activation
65 + in ["silu", "swish"])
66 + return (out, None) if not return_final_states else (out, final_states_out)
67 +
68 +
69 +def causal_conv1d_update(x: torch.Tensor,
70 + conv_state: torch.Tensor,
71 + weight: torch.Tensor,
72 + bias: Optional[torch.Tensor] = None,
73 + activation: Optional[str] = None):
74 + """
75 + x: (batch, dim)
76 + conv_state: (batch, dim, width)
77 + weight: (dim, width)
78 + bias: (dim,)
79 +
80 + out: (batch, dim)
81 + """
82 + if activation not in [None, "silu", "swish"]:
83 + raise NotImplementedError("activation must be None, silu, or swish")
84 + activation_bool = activation in ["silu", "swish"]
85 + return ops.causal_conv1d_update(x, conv_state, weight, bias,
86 + activation_bool)
vllm/model_executor/layers/mamba/ops/mamba_ssm.py new
+346
@@ -0,0 +1,346 @@
1 +# Copyright (c) 2024, Tri Dao, Albert Gu.
2 +
3 +import torch
4 +import triton
5 +import triton.language as tl
6 +from packaging import version
7 +
8 +from vllm import _custom_ops as ops
9 +
10 +TRITON3 = version.parse(triton.__version__) >= version.parse("3.0.0")
11 +
12 +if TRITON3:
13 +
14 + @triton.jit
15 + def softplus(dt):
16 + dt = tl.where(dt <= 20.0, tl.math.log(tl.math.exp(dt) + 1), dt)
17 + return dt
18 +else:
19 +
20 + @triton.jit
21 + def softplus(dt):
22 + dt = tl.where(dt <= 20.0, tl.math.log1p(tl.exp(dt)), dt)
23 + return dt
24 +
25 +
26 +@triton.heuristics(
27 + {"HAS_DT_BIAS": lambda args: args["dt_bias_ptr"] is not None})
28 +@triton.heuristics({"HAS_D": lambda args: args["D_ptr"] is not None})
29 +@triton.heuristics({"HAS_Z": lambda args: args["z_ptr"] is not None})
30 +@triton.heuristics(
31 + {"BLOCK_SIZE_DSTATE": lambda args: triton.next_power_of_2(args["dstate"])})
32 +@triton.jit
33 +def _selective_scan_update_kernel(
34 + # Pointers to matrices
35 + state_ptr,
36 + x_ptr,
37 + dt_ptr,
38 + dt_bias_ptr,
39 + A_ptr,
40 + B_ptr,
41 + C_ptr,
42 + D_ptr,
43 + z_ptr,
44 + out_ptr,
45 + # Matrix dimensions
46 + batch,
47 + nheads,
48 + dim,
49 + dstate,
50 + nheads_ngroups_ratio,
51 + # Strides
52 + stride_state_batch,
53 + stride_state_head,
54 + stride_state_dim,
55 + stride_state_dstate,
56 + stride_x_batch,
57 + stride_x_head,
58 + stride_x_dim,
59 + stride_dt_batch,
60 + stride_dt_head,
61 + stride_dt_dim,
62 + stride_dt_bias_head,
63 + stride_dt_bias_dim,
64 + stride_A_head,
65 + stride_A_dim,
66 + stride_A_dstate,
67 + stride_B_batch,
68 + stride_B_group,
69 + stride_B_dstate,
70 + stride_C_batch,
71 + stride_C_group,
72 + stride_C_dstate,
73 + stride_D_head,
74 + stride_D_dim,
75 + stride_z_batch,
76 + stride_z_head,
77 + stride_z_dim,
78 + stride_out_batch,
79 + stride_out_head,
80 + stride_out_dim,
81 + # Meta-parameters
82 + DT_SOFTPLUS: tl.constexpr,
83 + TIE_HDIM: tl.constexpr,
84 + BLOCK_SIZE_M: tl.constexpr,
85 + HAS_DT_BIAS: tl.constexpr,
86 + HAS_D: tl.constexpr,
87 + HAS_Z: tl.constexpr,
88 + BLOCK_SIZE_DSTATE: tl.constexpr,
89 +):
90 + pid_m = tl.program_id(axis=0)
91 + pid_b = tl.program_id(axis=1)
92 + pid_h = tl.program_id(axis=2)
93 + state_ptr += pid_b * stride_state_batch + pid_h * stride_state_head
94 + x_ptr += pid_b * stride_x_batch + pid_h * stride_x_head
95 + dt_ptr += pid_b * stride_dt_batch + pid_h * stride_dt_head
96 + if HAS_DT_BIAS:
97 + dt_bias_ptr += pid_h * stride_dt_bias_head
98 + A_ptr += pid_h * stride_A_head
99 + B_ptr += pid_b * stride_B_batch + (pid_h //
100 + nheads_ngroups_ratio) * stride_B_group
101 + C_ptr += pid_b * stride_C_batch + (pid_h //
102 + nheads_ngroups_ratio) * stride_C_group
103 + if HAS_Z:
104 + z_ptr += pid_b * stride_z_batch + pid_h * stride_z_head
105 + out_ptr += pid_b * stride_out_batch + pid_h * stride_out_head
106 +
107 + offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
108 + offs_n = tl.arange(0, BLOCK_SIZE_DSTATE)
109 + state_ptrs = state_ptr + (offs_m[:, None] * stride_state_dim +
110 + offs_n[None, :] * stride_state_dstate)
111 + x_ptrs = x_ptr + offs_m * stride_x_dim
112 + dt_ptrs = dt_ptr + offs_m * stride_dt_dim
113 + if HAS_DT_BIAS:
114 + dt_bias_ptrs = dt_bias_ptr + offs_m * stride_dt_bias_dim
115 + if HAS_D:
116 + D_ptr += pid_h * stride_D_head
117 + A_ptrs = A_ptr + (offs_m[:, None] * stride_A_dim +
118 + offs_n[None, :] * stride_A_dstate)
119 + B_ptrs = B_ptr + offs_n * stride_B_dstate
120 + C_ptrs = C_ptr + offs_n * stride_C_dstate
121 + if HAS_D:
122 + D_ptrs = D_ptr + offs_m * stride_D_dim
123 + if HAS_Z:
124 + z_ptrs = z_ptr + offs_m * stride_z_dim
125 + out_ptrs = out_ptr + offs_m * stride_out_dim
126 +
127 + state = tl.load(state_ptrs,
128 + mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate),
129 + other=0.0)
130 + x = tl.load(x_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
131 + if not TIE_HDIM:
132 + dt = tl.load(dt_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
133 + if HAS_DT_BIAS:
134 + dt += tl.load(dt_bias_ptrs, mask=offs_m < dim,
135 + other=0.0).to(tl.float32)
136 + if DT_SOFTPLUS:
137 + dt = softplus(dt)
138 + A = tl.load(A_ptrs,
139 + mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate),
140 + other=0.0).to(tl.float32)
141 + dA = tl.exp(A * dt[:, None])
142 + else:
143 + dt = tl.load(dt_ptr).to(tl.float32)
144 + if HAS_DT_BIAS:
145 + dt += tl.load(dt_bias_ptr).to(tl.float32)
146 + if DT_SOFTPLUS:
147 + dt = softplus(dt)
148 + A = tl.load(A_ptr).to(tl.float32)
149 + dA = tl.exp(A * dt) # scalar, not a matrix
150 +
151 + B = tl.load(B_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
152 + C = tl.load(C_ptrs, mask=offs_n < dstate, other=0.0).to(tl.float32)
153 + if HAS_D:
154 + D = tl.load(D_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
155 + if HAS_Z:
156 + z = tl.load(z_ptrs, mask=offs_m < dim, other=0.0).to(tl.float32)
157 +
158 + dB = B[None, :] * dt[:, None] if not TIE_HDIM else B * dt
159 + state = state * dA + dB * x[:, None]
160 + tl.store(state_ptrs,
161 + state,
162 + mask=(offs_m[:, None] < dim) & (offs_n[None, :] < dstate))
163 + out = tl.sum(state * C[None, :], axis=1)
164 + if HAS_D:
165 + out += x * D
166 + if HAS_Z:
167 + out *= z * tl.sigmoid(z)
168 + tl.store(out_ptrs, out, mask=offs_m < dim)
169 +
170 +
171 +def selective_state_update(state,
172 + x,
173 + dt,
174 + A,
175 + B,
176 + C,
177 + D=None,
178 + z=None,
179 + dt_bias=None,
180 + dt_softplus=False):
181 + """
182 + Argument:
183 + state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
184 + x: (batch, dim) or (batch, nheads, dim)
185 + dt: (batch, dim) or (batch, nheads, dim)
186 + A: (dim, dstate) or (nheads, dim, dstate)
187 + B: (batch, dstate) or (batch, ngroups, dstate)
188 + C: (batch, dstate) or (batch, ngroups, dstate)
189 + D: (dim,) or (nheads, dim)
190 + z: (batch, dim) or (batch, nheads, dim)
191 + dt_bias: (dim,) or (nheads, dim)
192 + Return:
193 + out: (batch, dim) or (batch, nheads, dim)
194 + """
195 + has_heads = state.dim() > 3
196 + if state.dim() == 3:
197 + state = state.unsqueeze(1)
198 + if x.dim() == 2:
199 + x = x.unsqueeze(1)
200 + if dt.dim() == 2:
201 + dt = dt.unsqueeze(1)
202 + if A.dim() == 2:
203 + A = A.unsqueeze(0)
204 + if B.dim() == 2:
205 + B = B.unsqueeze(1)
206 + if C.dim() == 2:
207 + C = C.unsqueeze(1)
208 + if D is not None and D.dim() == 1:
209 + D = D.unsqueeze(0)
210 + if z is not None and z.dim() == 2:
211 + z = z.unsqueeze(1)
212 + if dt_bias is not None and dt_bias.dim() == 1:
213 + dt_bias = dt_bias.unsqueeze(0)
214 + batch, nheads, dim, dstate = state.shape
215 + assert x.shape == (batch, nheads, dim)
216 + assert dt.shape == x.shape
217 + assert A.shape == (nheads, dim, dstate)
218 + ngroups = B.shape[1]
219 + assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
220 + assert B.shape == (batch, ngroups, dstate)
221 + assert C.shape == B.shape
222 + if D is not None:
223 + assert D.shape == (nheads, dim)
224 + if z is not None:
225 + assert z.shape == x.shape
226 + if dt_bias is not None:
227 + assert dt_bias.shape == (nheads, dim)
228 + out = torch.empty_like(x)
229 + grid = lambda META: (triton.cdiv(dim, META['BLOCK_SIZE_M']), batch, nheads)
230 + z_strides = ((z.stride(0), z.stride(1), z.stride(2)) if z is not None else
231 + (0, 0, 0))
232 + # We don't want autotune since it will overwrite the state
233 + # We instead tune by hand.
234 + BLOCK_SIZE_M, num_warps = ((32, 4) if dstate <= 16 else
235 + ((16, 4) if dstate <= 32 else
236 + ((8, 4) if dstate <= 64 else
237 + ((4, 4) if dstate <= 128 else ((4, 8))))))
238 + tie_hdim = A.stride(-1) == 0 and A.stride(-2) == 0 and dt.stride(
239 + -1) == 0 and dt_bias.stride(-1) == 0
240 + with torch.cuda.device(x.device.index):
241 + _selective_scan_update_kernel[grid](
242 + state,
243 + x,
244 + dt,
245 + dt_bias,
246 + A,
247 + B,
248 + C,
249 + D,
250 + z,
251 + out,
252 + batch,
253 + nheads,
254 + dim,
255 + dstate,
256 + nheads // ngroups,
257 + state.stride(0),
258 + state.stride(1),
259 + state.stride(2),
260 + state.stride(3),
261 + x.stride(0),
262 + x.stride(1),
263 + x.stride(2),
264 + dt.stride(0),
265 + dt.stride(1),
266 + dt.stride(2),
267 + *(dt_bias.stride(0),
268 + dt_bias.stride(1)) if dt_bias is not None else 0,
269 + A.stride(0),
270 + A.stride(1),
271 + A.stride(2),
272 + B.stride(0),
273 + B.stride(1),
274 + B.stride(2),
275 + C.stride(0),
276 + C.stride(1),
277 + C.stride(2),
278 + *(D.stride(0), D.stride(1)) if D is not None else 0,
279 + z_strides[0],
280 + z_strides[1],
281 + z_strides[2],
282 + out.stride(0),
283 + out.stride(1),
284 + out.stride(2),
285 + dt_softplus,
286 + tie_hdim,
287 + BLOCK_SIZE_M,
288 + num_warps=num_warps,
289 + )
290 + if not has_heads:
291 + out = out.squeeze(1)
292 + return out
293 +
294 +
295 +def selective_scan_fn(u,
296 + delta,
297 + A,
298 + B,
299 + C,
300 + D=None,
301 + z=None,
302 + delta_bias=None,
303 + delta_softplus=False,
304 + return_last_state=False,
305 + position_indices=None,
306 + prev_state=None):
307 + """if return_last_state is True, returns (out, last_state)
308 + last_state has shape (batch, dim, dstate).
309 + """
310 + if u.stride(-1) != 1:
311 + u = u.contiguous()
312 + if delta.stride(-1) != 1:
313 + delta = delta.contiguous()
314 + if D is not None:
315 + D = D.contiguous()
316 + if B.stride(-1) != 1:
317 + B = B.contiguous()
318 + if C.stride(-1) != 1:
319 + C = C.contiguous()
320 + if z is not None and z.stride(-1) != 1:
321 + z = z.contiguous()
322 + if B.dim() == 3:
323 + B = B.unsqueeze(1)
324 + if C.dim() == 3:
325 + C = C.unsqueeze(1)
326 + n_chunks = int((u.shape[-1] + 2048 - 1) / 2048)
327 + x = torch.zeros((
328 + u.shape[0],
329 + u.shape[1],
330 + n_chunks,
331 + int(A.shape[1] * 2),
332 + ),
333 + device=u.device,
334 + dtype=torch.float32,
335 + requires_grad=False)
336 + x[:, :, 0, 0::2] = 1
337 + if prev_state is not None:
338 + x[:, :, 0, 1::2].copy_(prev_state)
339 + out, x, *rest = ops.selective_scan_fwd(u, delta, A, B, C, D, z, delta_bias,
340 + delta_softplus, position_indices, x)
341 + last_state = x[:, :, -1, 1::2] # (batch, dim, dstate)
342 + if z is None:
343 + return out if not return_last_state else (out, last_state)
344 + else:
345 + out_z = rest[0]
346 + return out_z if not return_last_state else (out_z, last_state)
vllm/model_executor/models/jamba.py
+5 -4
@@ -4,9 +4,6 @@ from dataclasses import dataclass
4 from typing import Dict, Iterable, List, Optional, Tuple
5
6 import torch
7 -from causal_conv1d import causal_conv1d_fn, causal_conv1d_update
8 -from mamba_ssm.ops.selective_scan_interface import selective_scan_fn
9 -from mamba_ssm.ops.triton.selective_state_update import selective_state_update
7 from torch import nn
8 from torch.nn.parameter import Parameter
9 from transformers import JambaConfig
@@ -24,6 +21,10 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
21 ReplicatedLinear,
22 RowParallelLinear)
23 from vllm.model_executor.layers.logits_processor import LogitsProcessor
24 +from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
25 + causal_conv1d_fn, causal_conv1d_update)
26 +from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
27 + selective_scan_fn, selective_state_update)
28 from vllm.model_executor.layers.quantization.base_config import (
29 QuantizationConfig)
30 from vllm.model_executor.layers.sampler import Sampler
@@ -161,7 +162,7 @@ class JambaMambaMixer(nn.Module):
162 (self.conv_kernel_size - hidden_states.shape[-1], 0))
163 cache_params.conv_state.copy_(conv_states)
164
164 - hidden_states = causal_conv1d_fn(
165 + hidden_states, _ = causal_conv1d_fn(
166 hidden_states,
167 conv_weights,
168 self.conv1d.bias,