| 1 | #include <cudaTypedefs.h> |
| 2 | |
| 3 | #include "libtorch_stable/torch_utils.h" |
| 4 | #include <torch/csrc/stable/tensor.h> |
| 5 | #include <torch/csrc/stable/ops.h> |
| 6 | #include <torch/headeronly/core/ScalarType.h> |
| 7 | |
| 8 | #include "libtorch_stable/dispatch_utils.h" |
| 9 | |
| 10 | #include <iostream> |
| 11 | |
| 12 | constexpr uint64_t THREADS_PER_EXPERT = 512; |
| 13 | // threshold must match the dispatch logic in run_cutlass_moe_mm_sm90() |
| 14 | constexpr int SWAP_AB_THRESHOLD = 64; |
| 15 | |
| 16 | template <bool SWAP_AB> |
| 17 | __global__ void compute_problem_sizes(const int32_t* __restrict__ topk_ids, |
| 18 | int32_t* problem_sizes1, |
| 19 | int32_t* problem_sizes2, |
| 20 | int32_t* atomic_buffer, |
| 21 | const int topk_length, const int n, |
| 22 | const int k, const bool is_gated) { |
| 23 | int expert_id = blockIdx.x; |
| 24 | // For gated activations (gate + up), first GEMM output is 2*n. |
| 25 | // For non-gated activations (up only), first GEMM output is n. |
| 26 | int const n1 = is_gated ? 2 * n : n; |
| 27 | |
| 28 | int occurrences = 0; |
| 29 | for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { |
| 30 | occurrences += (topk_ids[i] == expert_id); |
| 31 | } |
| 32 | atomicAdd(&atomic_buffer[expert_id], occurrences); |
| 33 | __syncthreads(); |
| 34 | |
| 35 | if (threadIdx.x == 0) { |
| 36 | int final_occurrences = atomic_buffer[expert_id]; |
| 37 | if constexpr (!SWAP_AB) { |
| 38 | problem_sizes1[expert_id * 3] = final_occurrences; |
| 39 | problem_sizes1[expert_id * 3 + 1] = n1; |
| 40 | problem_sizes1[expert_id * 3 + 2] = k; |
| 41 | problem_sizes2[expert_id * 3] = final_occurrences; |
| 42 | problem_sizes2[expert_id * 3 + 1] = k; |
| 43 | problem_sizes2[expert_id * 3 + 2] = n; |
| 44 | } else { |
| 45 | problem_sizes1[expert_id * 3] = n1; |
| 46 | problem_sizes1[expert_id * 3 + 1] = final_occurrences; |
| 47 | problem_sizes1[expert_id * 3 + 2] = k; |
| 48 | problem_sizes2[expert_id * 3] = k; |
| 49 | problem_sizes2[expert_id * 3 + 1] = final_occurrences; |
| 50 | problem_sizes2[expert_id * 3 + 2] = n; |
| 51 | } |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | __global__ void compute_expert_offsets( |
| 56 | const int32_t* __restrict__ problem_sizes1, int32_t* expert_offsets, |
| 57 | int32_t* atomic_buffer, const int num_experts, const bool swap_ab) { |
| 58 | int32_t tot_offset = 0; |
| 59 | expert_offsets[0] = 0; |
| 60 | for (int i = 0; i < num_experts; ++i) { |
| 61 | atomic_buffer[i] = tot_offset; |
| 62 | tot_offset += swap_ab ? problem_sizes1[i * 3 + 1] : problem_sizes1[i * 3]; |
| 63 | expert_offsets[i + 1] = tot_offset; |
| 64 | } |
| 65 | atomic_buffer[num_experts] = tot_offset; |
| 66 | } |
| 67 | |
| 68 | __global__ void compute_expert_blockscale_offsets( |
| 69 | const int32_t* __restrict__ problem_sizes1, int32_t* expert_offsets, |
| 70 | int32_t* blockscale_offsets, int32_t* atomic_buffer, const int num_experts, |
| 71 | const bool swap_ab) { |
| 72 | int32_t tot_offset = 0; |
| 73 | int32_t tot_offset_round = 0; |
| 74 | expert_offsets[0] = 0; |
| 75 | blockscale_offsets[0] = 0; |
| 76 | for (int i = 0; i < num_experts; ++i) { |
| 77 | int32_t cur_offset = |
| 78 | swap_ab ? problem_sizes1[i * 3 + 1] : problem_sizes1[i * 3]; |
| 79 | atomic_buffer[i] = tot_offset; |
| 80 | tot_offset += cur_offset; |
| 81 | expert_offsets[i + 1] = tot_offset; |
| 82 | tot_offset_round += (cur_offset + (128 - 1)) / 128 * 128; |
| 83 | blockscale_offsets[i + 1] = tot_offset_round; |
| 84 | } |
| 85 | atomic_buffer[num_experts] = tot_offset; |
| 86 | } |
| 87 | |
| 88 | __global__ void compute_arg_sorts(const int32_t* __restrict__ topk_ids, |
| 89 | int32_t* input_permutation, |
| 90 | int32_t* output_permutation, |
| 91 | int32_t* atomic_buffer, const int topk_length, |
| 92 | const int topk) { |
| 93 | int const blk_expert_id = blockIdx.x; |
| 94 | int const num_experts = gridDim.x; |
| 95 | |
| 96 | for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { |
| 97 | int const expert_id = topk_ids[i]; |
| 98 | if ((expert_id < 0 || expert_id >= num_experts) && blockIdx.x == 0) { |
| 99 | int const start = atomicAdd(&atomic_buffer[num_experts], 1); |
| 100 | input_permutation[start] = -1; |
| 101 | // output_permutation is used to re-order the moe outputs. It is |
| 102 | // used as c2 = c2[c_map], where c2 is a torch.tensor that is the |
| 103 | // output of the cutlass kernels and c_map is the output_permutation. Use |
| 104 | // out-of-range sentinels for invalid routes; shuffle_rows converts them |
| 105 | // to zero rows. |
| 106 | output_permutation[i] = topk_length; |
| 107 | } else if (expert_id == blk_expert_id) { |
| 108 | int start = atomicAdd(&atomic_buffer[expert_id], 1); |
| 109 | input_permutation[start] = i / topk; |
| 110 | output_permutation[i] = start; |
| 111 | } |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | namespace { |
| 116 | inline void launch_compute_problem_sizes(const torch::stable::Tensor& topk_ids, |
| 117 | torch::stable::Tensor& problem_sizes1, |
| 118 | torch::stable::Tensor& problem_sizes2, |
| 119 | torch::stable::Tensor& atomic_buffer, |
| 120 | int64_t num_experts, int64_t n, |
| 121 | int64_t k, cudaStream_t stream, |
| 122 | const bool swap_ab, |
| 123 | const bool is_gated) { |
| 124 | int num_threads = min(THREADS_PER_EXPERT, topk_ids.numel()); |
| 125 | |
| 126 | auto const* topk_ptr = topk_ids.const_data_ptr<int32_t>(); |
| 127 | auto* ps1_ptr = problem_sizes1.mutable_data_ptr<int32_t>(); |
| 128 | auto* ps2_ptr = problem_sizes2.mutable_data_ptr<int32_t>(); |
| 129 | auto* atomic_ptr = atomic_buffer.mutable_data_ptr<int32_t>(); |
| 130 | |
| 131 | VLLM_STABLE_DISPATCH_BOOL(swap_ab, SwapAB, [&] { |
| 132 | compute_problem_sizes<SwapAB><<<num_experts, num_threads, 0, stream>>>( |
| 133 | topk_ptr, ps1_ptr, ps2_ptr, atomic_ptr, |
| 134 | static_cast<int>(topk_ids.numel()), static_cast<int>(n), |
| 135 | static_cast<int>(k), is_gated); |
| 136 | }); |
| 137 | } |
| 138 | } // namespace |
| 139 | |
| 140 | template <bool SWAP_AB> |
| 141 | __global__ void compute_problem_sizes_from_expert_offsets( |
| 142 | const int64_t* __restrict__ expert_first_token_offset, |
| 143 | int32_t* __restrict__ problem_sizes1, int32_t* __restrict__ problem_sizes2, |
| 144 | const int num_experts, const int n, const int k) { |
| 145 | int const expert_id = blockIdx.x * blockDim.x + threadIdx.x; |
| 146 | if (expert_id >= num_experts) { |
| 147 | return; |
| 148 | } |
| 149 | |
| 150 | int64_t const m64 = expert_first_token_offset[expert_id + 1] - |
| 151 | expert_first_token_offset[expert_id]; |
| 152 | int32_t const m = static_cast<int32_t>(m64); |
| 153 | |
| 154 | int32_t* ps1 = problem_sizes1 + expert_id * 3; |
| 155 | int32_t* ps2 = problem_sizes2 + expert_id * 3; |
| 156 | |
| 157 | if constexpr (!SWAP_AB) { |
| 158 | // [M, 2*N, K] |
| 159 | ps1[0] = m; |
| 160 | ps1[1] = 2 * n; |
| 161 | ps1[2] = k; |
| 162 | // [M, K, N] |
| 163 | ps2[0] = m; |
| 164 | ps2[1] = k; |
| 165 | ps2[2] = n; |
| 166 | } else { |
| 167 | // swap logical M/N in the problem shape |
| 168 | // [2*N, M, K] |
| 169 | ps1[0] = 2 * n; |
| 170 | ps1[1] = m; |
| 171 | ps1[2] = k; |
| 172 | // [K, M, N] |
| 173 | ps2[0] = k; |
| 174 | ps2[1] = m; |
| 175 | ps2[2] = n; |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | void get_cutlass_moe_mm_problem_sizes_from_expert_offsets_caller( |
| 180 | const torch::stable::Tensor& expert_first_token_offset, |
| 181 | torch::stable::Tensor& problem_sizes1, |
| 182 | torch::stable::Tensor& problem_sizes2, const int64_t n, const int64_t k, |
| 183 | const bool swap_ab) { |
| 184 | STD_TORCH_CHECK(expert_first_token_offset.is_cuda(), |
| 185 | "expert_first_token_offset must be a CUDA tensor"); |
| 186 | STD_TORCH_CHECK(expert_first_token_offset.scalar_type() == |
| 187 | torch::headeronly::ScalarType::Long, |
| 188 | "expert_first_token_offset must be int64"); |
| 189 | |
| 190 | STD_TORCH_CHECK(problem_sizes1.is_cuda() && problem_sizes2.is_cuda(), |
| 191 | "problem_sizes must be CUDA tensors"); |
| 192 | STD_TORCH_CHECK( |
| 193 | problem_sizes1.scalar_type() == torch::headeronly::ScalarType::Int && |
| 194 | problem_sizes2.scalar_type() == torch::headeronly::ScalarType::Int, |
| 195 | "problem_sizes must be int32"); |
| 196 | STD_TORCH_CHECK( |
| 197 | problem_sizes1.is_contiguous() && problem_sizes2.is_contiguous(), |
| 198 | "problem_sizes must be contiguous"); |
| 199 | STD_TORCH_CHECK(problem_sizes1.dim() == 2 && problem_sizes2.dim() == 2, |
| 200 | "problem_sizes must be 2D tensors"); |
| 201 | STD_TORCH_CHECK(problem_sizes1.size(1) == 3 && problem_sizes2.size(1) == 3, |
| 202 | "problem_sizes second dim must be 3"); |
| 203 | STD_TORCH_CHECK(problem_sizes1.size(0) == problem_sizes2.size(0) && |
| 204 | problem_sizes1.size(1) == problem_sizes2.size(1), |
| 205 | "problem_sizes1 and problem_sizes2 must have same shape"); |
| 206 | |
| 207 | int64_t const num_experts64 = problem_sizes1.size(0); |
| 208 | STD_TORCH_CHECK( |
| 209 | expert_first_token_offset.numel() == num_experts64 + 1, |
| 210 | "expert_first_token_offset must have num_experts + 1 elements"); |
| 211 | STD_TORCH_CHECK(num_experts64 <= INT32_MAX, "num_experts must fit in int32"); |
| 212 | STD_TORCH_CHECK(n <= INT32_MAX && k <= INT32_MAX, |
| 213 | "n and k must fit in int32"); |
| 214 | |
| 215 | int const num_experts = static_cast<int>(num_experts64); |
| 216 | const torch::stable::accelerator::DeviceGuard device_guard( |
| 217 | expert_first_token_offset.get_device_index()); |
| 218 | auto stream = |
| 219 | get_current_cuda_stream(expert_first_token_offset.get_device_index()); |
| 220 | |
| 221 | int const threads = (num_experts < 256) ? num_experts : 256; |
| 222 | int const blocks = (num_experts + threads - 1) / threads; |
| 223 | |
| 224 | auto const* offsets_ptr = expert_first_token_offset.const_data_ptr<int64_t>(); |
| 225 | auto* ps1_ptr = problem_sizes1.mutable_data_ptr<int32_t>(); |
| 226 | auto* ps2_ptr = problem_sizes2.mutable_data_ptr<int32_t>(); |
| 227 | |
| 228 | VLLM_STABLE_DISPATCH_BOOL(swap_ab, SwapAB, [&] { |
| 229 | compute_problem_sizes_from_expert_offsets<SwapAB> |
| 230 | <<<blocks, threads, 0, stream>>>(offsets_ptr, ps1_ptr, ps2_ptr, |
| 231 | num_experts, static_cast<int>(n), |
| 232 | static_cast<int>(k)); |
| 233 | }); |
| 234 | } |
| 235 | |
| 236 | void get_cutlass_moe_mm_data_caller( |
| 237 | const torch::stable::Tensor& topk_ids, |
| 238 | torch::stable::Tensor& expert_offsets, |
| 239 | torch::stable::Tensor& problem_sizes1, |
| 240 | torch::stable::Tensor& problem_sizes2, |
| 241 | torch::stable::Tensor& input_permutation, |
| 242 | torch::stable::Tensor& output_permutation, const int64_t num_experts, |
| 243 | const int64_t n, const int64_t k, |
| 244 | const std::optional<torch::stable::Tensor>& blockscale_offsets, |
| 245 | const bool is_gated) { |
| 246 | auto device = topk_ids.device(); |
| 247 | const torch::stable::accelerator::DeviceGuard device_guard(device.index()); |
| 248 | auto stream = get_current_cuda_stream(device.index()); |
| 249 | torch::stable::Tensor atomic_buffer = torch::stable::new_zeros( |
| 250 | topk_ids, {num_experts + 1}, torch::headeronly::ScalarType::Int); |
| 251 | |
| 252 | int num_threads = min(THREADS_PER_EXPERT, topk_ids.numel()); |
| 253 | |
| 254 | // Swap-AB should be disabled for FP4 path |
| 255 | bool may_swap_ab = (!blockscale_offsets.has_value()) && |
| 256 | (topk_ids.numel() <= SWAP_AB_THRESHOLD); |
| 257 | |
| 258 | launch_compute_problem_sizes(topk_ids, problem_sizes1, problem_sizes2, |
| 259 | atomic_buffer, num_experts, n, k, stream, |
| 260 | may_swap_ab, is_gated); |
| 261 | |
| 262 | if (blockscale_offsets.has_value()) { |
| 263 | // fp4 path |
| 264 | compute_expert_blockscale_offsets<<<1, 1, 0, stream>>>( |
| 265 | static_cast<const int32_t*>(problem_sizes1.data_ptr()), |
| 266 | static_cast<int32_t*>(expert_offsets.data_ptr()), |
| 267 | static_cast<int32_t*>(blockscale_offsets.value().data_ptr()), |
| 268 | static_cast<int32_t*>(atomic_buffer.data_ptr()), num_experts, |
| 269 | may_swap_ab); |
| 270 | } else { |
| 271 | compute_expert_offsets<<<1, 1, 0, stream>>>( |
| 272 | static_cast<const int32_t*>(problem_sizes1.data_ptr()), |
| 273 | static_cast<int32_t*>(expert_offsets.data_ptr()), |
| 274 | static_cast<int32_t*>(atomic_buffer.data_ptr()), num_experts, |
| 275 | may_swap_ab); |
| 276 | } |
| 277 | compute_arg_sorts<<<num_experts, num_threads, 0, stream>>>( |
| 278 | static_cast<const int32_t*>(topk_ids.data_ptr()), |
| 279 | static_cast<int32_t*>(input_permutation.data_ptr()), |
| 280 | static_cast<int32_t*>(output_permutation.data_ptr()), |
| 281 | static_cast<int32_t*>(atomic_buffer.data_ptr()), topk_ids.numel(), |
| 282 | topk_ids.size(1)); |
| 283 | } |
| 284 | |
| 285 | template <bool SWAP_AB> |
| 286 | __global__ void compute_batched_moe_data( |
| 287 | int32_t* expert_offsets, int32_t* problem_sizes1, int32_t* problem_sizes2, |
| 288 | const int32_t* __restrict__ expert_num_tokens, const int padded_m, |
| 289 | const int n, const int k) { |
| 290 | int expert_idx = threadIdx.x; |
| 291 | expert_offsets[expert_idx] = expert_idx * padded_m; |
| 292 | |
| 293 | if constexpr (!SWAP_AB) { |
| 294 | problem_sizes1[expert_idx * 3] = expert_num_tokens[expert_idx]; |
| 295 | problem_sizes1[expert_idx * 3 + 1] = 2 * n; |
| 296 | problem_sizes1[expert_idx * 3 + 2] = k; |
| 297 | problem_sizes2[expert_idx * 3] = expert_num_tokens[expert_idx]; |
| 298 | problem_sizes2[expert_idx * 3 + 1] = k; |
| 299 | problem_sizes2[expert_idx * 3 + 2] = n; |
| 300 | } else { |
| 301 | problem_sizes1[expert_idx * 3] = 2 * n; |
| 302 | problem_sizes1[expert_idx * 3 + 1] = expert_num_tokens[expert_idx]; |
| 303 | problem_sizes1[expert_idx * 3 + 2] = k; |
| 304 | problem_sizes2[expert_idx * 3] = k; |
| 305 | problem_sizes2[expert_idx * 3 + 1] = expert_num_tokens[expert_idx]; |
| 306 | problem_sizes2[expert_idx * 3 + 2] = n; |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | void get_cutlass_batched_moe_mm_data_caller( |
| 311 | torch::stable::Tensor& expert_offsets, |
| 312 | torch::stable::Tensor& problem_sizes1, |
| 313 | torch::stable::Tensor& problem_sizes2, |
| 314 | const torch::stable::Tensor& expert_num_tokens, |
| 315 | const int64_t num_local_experts, const int64_t padded_m, const int64_t n, |
| 316 | const int64_t k) { |
| 317 | const torch::stable::accelerator::DeviceGuard device_guard( |
| 318 | expert_offsets.get_device_index()); |
| 319 | auto stream = get_current_cuda_stream(expert_offsets.get_device_index()); |
| 320 | |
| 321 | if (num_local_experts * padded_m > SWAP_AB_THRESHOLD) { |
| 322 | compute_batched_moe_data<false><<<1, num_local_experts, 0, stream>>>( |
| 323 | static_cast<int32_t*>(expert_offsets.data_ptr()), |
| 324 | static_cast<int32_t*>(problem_sizes1.data_ptr()), |
| 325 | static_cast<int32_t*>(problem_sizes2.data_ptr()), |
| 326 | static_cast<const int32_t*>(expert_num_tokens.data_ptr()), padded_m, n, |
| 327 | k); |
| 328 | } else { |
| 329 | compute_batched_moe_data<true><<<1, num_local_experts, 0, stream>>>( |
| 330 | static_cast<int32_t*>(expert_offsets.data_ptr()), |
| 331 | static_cast<int32_t*>(problem_sizes1.data_ptr()), |
| 332 | static_cast<int32_t*>(problem_sizes2.data_ptr()), |
| 333 | static_cast<const int32_t*>(expert_num_tokens.data_ptr()), padded_m, n, |
| 334 | k); |
| 335 | } |
| 336 | } |