| 1 | |
| 2 | /* |
| 3 | * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. |
| 4 | * |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | */ |
| 17 | |
| 18 | #include "torch_utils.h" |
| 19 | |
| 20 | #include <torch/csrc/stable/macros.h> |
| 21 | #include <torch/csrc/stable/accelerator.h> |
| 22 | #include <torch/csrc/stable/tensor.h> |
| 23 | #include <torch/csrc/stable/ops.h> |
| 24 | #include <torch/headeronly/core/ScalarType.h> |
| 25 | #include <torch/csrc/stable/device.h> |
| 26 | |
| 27 | #include <cooperative_groups.h> |
| 28 | #include <cuda_runtime.h> |
| 29 | |
| 30 | #include "cuda_compat.h" |
| 31 | #include "minimax_reduce_rms_kernel.h" |
| 32 | |
| 33 | #include <algorithm> |
| 34 | |
| 35 | #define FINAL_MASK 0xffffffff |
| 36 | #define MINIMAX_REDUCE_RMS_WARP_SIZE 32 |
| 37 | |
| 38 | namespace vllm { |
| 39 | namespace tensorrt_llm { |
| 40 | |
| 41 | template <int NRanks> |
| 42 | struct LamportComm { |
| 43 | __device__ __forceinline__ LamportComm(void** workspace, int rank) { |
| 44 | counter_ptr = &reinterpret_cast<int*>(workspace[NRanks * 3])[0]; |
| 45 | flag_ptr = &reinterpret_cast<int*>(workspace[NRanks * 3])[2]; |
| 46 | clear_ptr = &reinterpret_cast<int64_t*>(workspace[NRanks * 3 + 1])[0]; |
| 47 | flag_value = *flag_ptr; |
| 48 | auto comm_size = reinterpret_cast<int64_t*>(workspace[NRanks * 3 + 1])[1]; |
| 49 | clear_size = *clear_ptr; |
| 50 | int data_offset = flag_value % 3; |
| 51 | int clear_offset = (flag_value + 2) % 3; |
| 52 | for (int r = 0; r < NRanks; ++r) { |
| 53 | data_bufs[r] = reinterpret_cast<uint8_t*>(workspace[2 * NRanks + r]) + |
| 54 | data_offset * comm_size; |
| 55 | } |
| 56 | clear_buf = reinterpret_cast<uint8_t*>(workspace[2 * NRanks + rank]) + |
| 57 | clear_offset * comm_size; |
| 58 | __syncthreads(); |
| 59 | if (threadIdx.x == 0) { |
| 60 | atomicAdd(counter_ptr, 1); |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | __device__ __forceinline__ void update(int64_t new_clear_size) { |
| 65 | if (blockIdx.x == 0 && threadIdx.x == 0) { |
| 66 | while (*reinterpret_cast<int volatile*>(counter_ptr) != gridDim.x) { |
| 67 | } |
| 68 | *flag_ptr = (flag_value + 1) % 3; |
| 69 | *clear_ptr = new_clear_size; |
| 70 | *counter_ptr = 0; |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | int* counter_ptr; |
| 75 | int* flag_ptr; |
| 76 | int64_t* clear_ptr; |
| 77 | uint8_t* data_bufs[NRanks]; |
| 78 | uint8_t* clear_buf; |
| 79 | int64_t clear_size; |
| 80 | int flag_value; |
| 81 | }; |
| 82 | |
| 83 | __device__ __forceinline__ bool is_neg_zero(float v) { |
| 84 | return *reinterpret_cast<uint32_t*>(&v) == 0x80000000; |
| 85 | } |
| 86 | |
| 87 | __device__ __forceinline__ bool is_neg_zero(float4 v) { |
| 88 | return is_neg_zero(v.x) || is_neg_zero(v.y) || is_neg_zero(v.z) || |
| 89 | is_neg_zero(v.w); |
| 90 | } |
| 91 | |
| 92 | __device__ __forceinline__ float4 get_neg_zero() { |
| 93 | float4 vec; |
| 94 | #pragma unroll |
| 95 | for (int i = 0; i < 4; ++i) { |
| 96 | reinterpret_cast<uint32_t*>(&vec)[i] = 0x80000000; |
| 97 | } |
| 98 | return vec; |
| 99 | } |
| 100 | |
| 101 | template <int Dim> |
| 102 | __device__ __forceinline__ float rms_rsqrt(float& v, float eps) { |
| 103 | constexpr float kInvDim = 1.0F / static_cast<float>(Dim); |
| 104 | v = rsqrtf((v * kInvDim) + eps); |
| 105 | return v; |
| 106 | } |
| 107 | |
| 108 | template <int Dim> |
| 109 | __device__ __forceinline__ float4 rms_rsqrt(float4& v, float eps) { |
| 110 | constexpr float kInvDim = 1.0F / static_cast<float>(Dim); |
| 111 | v.x = rsqrtf((v.x * kInvDim) + eps); |
| 112 | v.y = rsqrtf((v.y * kInvDim) + eps); |
| 113 | v.z = rsqrtf((v.z * kInvDim) + eps); |
| 114 | v.w = rsqrtf((v.w * kInvDim) + eps); |
| 115 | return v; |
| 116 | } |
| 117 | __device__ __forceinline__ float4 ld_global_volatile(float4* addr) { |
| 118 | float4 val; |
| 119 | asm volatile("ld.volatile.global.v4.f32 {%0, %1, %2, %3}, [%4];" |
| 120 | : "=f"(val.x), "=f"(val.y), "=f"(val.z), "=f"(val.w) |
| 121 | : "l"(addr)); |
| 122 | return val; |
| 123 | } |
| 124 | |
| 125 | __device__ __forceinline__ float ld_global_volatile(float* addr) { |
| 126 | float val; |
| 127 | asm volatile("ld.volatile.global.f32 %0, [%1];" : "=f"(val) : "l"(addr)); |
| 128 | return val; |
| 129 | } |
| 130 | |
| 131 | // Used by the scalar (non-float4) kernel only |
| 132 | template <typename T, int NUM> |
| 133 | __inline__ __device__ T warpReduceSumV2(T* val) { |
| 134 | #pragma unroll |
| 135 | for (int i = 0; i < NUM; i++) { |
| 136 | #pragma unroll |
| 137 | for (int mask = 16; mask > 0; mask >>= 1) |
| 138 | val[i] += __shfl_xor_sync(FINAL_MASK, val[i], mask, 32); |
| 139 | } |
| 140 | return (T)(0.0f); |
| 141 | } |
| 142 | |
| 143 | template <typename T, int NUM> |
| 144 | __inline__ __device__ T blockReduceSumV2(T* val) { |
| 145 | static __shared__ T shared[NUM][33]; |
| 146 | int lane = threadIdx.x & 0x1f; |
| 147 | int wid = threadIdx.x >> 5; |
| 148 | |
| 149 | warpReduceSumV2<T, NUM>(val); |
| 150 | |
| 151 | if (lane == 0) { |
| 152 | #pragma unroll |
| 153 | for (int i = 0; i < NUM; i++) { |
| 154 | shared[i][wid] = val[i]; |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | __syncthreads(); |
| 159 | |
| 160 | bool is_mask = threadIdx.x < (blockDim.x / 32.f); |
| 161 | #pragma unroll |
| 162 | for (int i = 0; i < NUM; i++) { |
| 163 | val[i] = is_mask ? shared[i][lane] : (T)(0.0f); |
| 164 | } |
| 165 | warpReduceSumV2<T, NUM>(val); |
| 166 | return (T)0.0f; |
| 167 | } |
| 168 | |
| 169 | // for float4 version |
| 170 | template <uint32_t kNumThreads, typename T, int ArraySize = 4> |
| 171 | __device__ __forceinline__ void local_warp_reduce_sum_array( |
| 172 | T* value_ptr, uint32_t active_mask = 0xffffffffu) { |
| 173 | static_assert(kNumThreads >= 1 && |
| 174 | kNumThreads <= MINIMAX_REDUCE_RMS_WARP_SIZE); |
| 175 | #pragma unroll |
| 176 | for (int i = 0; i < ArraySize; ++i) { |
| 177 | #pragma unroll |
| 178 | for (int mask = kNumThreads / 2; mask > 0; mask >>= 1) { |
| 179 | value_ptr[i] += __shfl_xor_sync(active_mask, value_ptr[i], mask, |
| 180 | MINIMAX_REDUCE_RMS_WARP_SIZE); |
| 181 | } |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | constexpr int next_pow2(int val) { |
| 186 | int result = 1; |
| 187 | while (result < val) { |
| 188 | result <<= 1; |
| 189 | } |
| 190 | return result; |
| 191 | } |
| 192 | |
| 193 | // --------------------------------------------------------------------------- |
| 194 | |
| 195 | template <typename DType> |
| 196 | class IndexHelper { |
| 197 | public: |
| 198 | __device__ __forceinline__ IndexHelper(MiniMaxReduceRMSParams const& params) { |
| 199 | #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) |
| 200 | namespace cg = cooperative_groups; |
| 201 | cg::cluster_group cluster = cg::this_cluster(); |
| 202 | cg::grid_group grid = cg::this_grid(); |
| 203 | token_id = grid.cluster_rank(); |
| 204 | access_id_in_token = cluster.thread_rank(); |
| 205 | token_stride = grid.num_clusters(); |
| 206 | #else |
| 207 | token_id = blockIdx.x; |
| 208 | access_id_in_token = threadIdx.x; |
| 209 | token_stride = gridDim.x; |
| 210 | #endif |
| 211 | access_id = token_id * params.hidden_dim / kElemsPerAccess<DType> + |
| 212 | access_id_in_token; |
| 213 | access_stride = token_stride * params.hidden_dim / kElemsPerAccess<DType>; |
| 214 | tot_access = params.size_q / kElemsPerAccess<DType>; |
| 215 | } |
| 216 | |
| 217 | int token_id; |
| 218 | int access_id_in_token; |
| 219 | int token_stride; |
| 220 | int access_id; |
| 221 | int access_stride; |
| 222 | int tot_access; |
| 223 | }; |
| 224 | |
| 225 | /** |
| 226 | * this kernel is used to for minimax attention module |
| 227 | * input tensor [total_tokens, hidden_dim / tp_size], fp32 |
| 228 | * rms weight [hidden_dim / tp_size], bf16 |
| 229 | step 1: reduce from single rank to get the variance sum (reduce(input^2, |
| 230 | dim=-1)) step 2: reduce from all ranks to get the variance sum |
| 231 | (all_reduce(variance_sum)) step 3: calculate the rms norm (input * |
| 232 | rsqrt(variance + eps)) in this case, max hidden_dim is 6144 (float data), for |
| 233 | each token, we only need 6144 / 4 / tp_size = (1536 / tp_size) threads so we can |
| 234 | assume cluster size is 1 (tp_size >= 2) |
| 235 | */ |
| 236 | template <typename DType, int NRanks> |
| 237 | __global__ void __launch_bounds__(1024) |
| 238 | minimax_reduce_rms_kernel_lamport(MiniMaxReduceRMSParams params) { |
| 239 | IndexHelper<DType> index_helper(params); |
| 240 | int token_id = index_helper.token_id; |
| 241 | int access_id_in_token = index_helper.access_id_in_token; |
| 242 | int token_stride = index_helper.token_stride; |
| 243 | int access_id = index_helper.access_id; |
| 244 | int access_stride = index_helper.access_stride; |
| 245 | int tot_access = index_helper.tot_access; |
| 246 | int tot_tokens = params.size_q / params.hidden_dim; |
| 247 | float4 clear_vec = get_neg_zero(); |
| 248 | |
| 249 | LamportComm<NRanks> comm(params.workspace, params.rank); |
| 250 | int clear_access = comm.clear_size / kElemsPerAccess<DType>; |
| 251 | #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) |
| 252 | cudaGridDependencySynchronize(); |
| 253 | #endif |
| 254 | for (int idx = access_id; idx < tot_access; |
| 255 | idx += access_stride, token_id += token_stride) { |
| 256 | alignas(16) DType vals[kElemsPerAccess<DType>]; |
| 257 | float sum_variance = 0.F; |
| 258 | *reinterpret_cast<float4*>(vals) = |
| 259 | reinterpret_cast<float4*>(params.allreduce_in)[idx]; |
| 260 | #pragma unroll |
| 261 | for (int i = 0; i < kElemsPerAccess<DType>; ++i) { |
| 262 | sum_variance += static_cast<float>(vals[i]) * static_cast<float>(vals[i]); |
| 263 | } |
| 264 | blockReduceSumV2<float, 1>(&sum_variance); |
| 265 | if (is_neg_zero(sum_variance)) { |
| 266 | sum_variance = 0.F; |
| 267 | } |
| 268 | if (threadIdx.x == 0) { |
| 269 | for (int r = 0; r < NRanks; ++r) { |
| 270 | reinterpret_cast<float*>( |
| 271 | comm.data_bufs[r])[(params.rank * tot_tokens) + token_id] = |
| 272 | (sum_variance); |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | bool done = false; |
| 277 | float vars_all_ranks[NRanks]; |
| 278 | while (!done) { |
| 279 | done = true; |
| 280 | #pragma unroll |
| 281 | for (int r = 0; r < NRanks; ++r) { |
| 282 | vars_all_ranks[r] = ld_global_volatile(&reinterpret_cast<float*>( |
| 283 | comm.data_bufs[params.rank])[(r * tot_tokens) + token_id]); |
| 284 | done &= !is_neg_zero(vars_all_ranks[r]); |
| 285 | } |
| 286 | } |
| 287 | sum_variance = 0.F; |
| 288 | #pragma unroll |
| 289 | for (int r = 0; r < NRanks; ++r) { |
| 290 | sum_variance += vars_all_ranks[r]; |
| 291 | } |
| 292 | |
| 293 | DType norm_weight[kElemsPerAccess<DType>]; |
| 294 | *reinterpret_cast<typename ElemsPerAccess<DType>::vec_type*>(norm_weight) = |
| 295 | reinterpret_cast<typename ElemsPerAccess<DType>::vec_type*>( |
| 296 | params.rms_gamma)[access_id_in_token]; |
| 297 | |
| 298 | #pragma unroll |
| 299 | for (int i = 0; i < kElemsPerAccess<DType>; ++i) { |
| 300 | vals[i] = static_cast<DType>( |
| 301 | static_cast<float>(vals[i]) * |
| 302 | rsqrtf( |
| 303 | (sum_variance / static_cast<float>(params.hidden_dim) / NRanks) + |
| 304 | params.rms_eps) * |
| 305 | static_cast<float>(norm_weight[i])); |
| 306 | } |
| 307 | |
| 308 | reinterpret_cast<float4*>(params.rms_norm_out)[idx] = |
| 309 | *reinterpret_cast<float4*>(vals); |
| 310 | } |
| 311 | for (int idx = access_id; idx < clear_access; idx += access_stride) { |
| 312 | reinterpret_cast<float4*>(comm.clear_buf)[idx] = clear_vec; |
| 313 | } |
| 314 | comm.update(params.size_q * NRanks); |
| 315 | #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) |
| 316 | cudaTriggerProgrammaticLaunchCompletion(); |
| 317 | #endif |
| 318 | } |
| 319 | |
| 320 | /** |
| 321 | * Float4 variant: process 4 rows at once, allreduce variance sums as float4 for |
| 322 | * better memory coalescing. sum_variance is always float; applies to all DTypes |
| 323 | * (half, bf16, float). When tot_tokens % 4 != 0, the last group pads rows with |
| 324 | * zeros; padded rows are not written to rms_norm_out. IsQK: when true, process |
| 325 | * Q+K in one loop with doubled comm buffer; when false, single-matrix (Q only). |
| 326 | */ |
| 327 | template <typename DType, int NRanks, int OriginQDim, int OriginKDim> |
| 328 | __global__ void __launch_bounds__(1024) |
| 329 | minimax_reduce_qk_rms_kernel_lamport_float4(MiniMaxReduceRMSParams params) { |
| 330 | // Compile-time per-rank dimensions |
| 331 | constexpr int RankQDim = OriginQDim / NRanks; |
| 332 | constexpr int RankKDim = OriginKDim / NRanks; |
| 333 | // Threads needed to cover one row of Q / K with float4 accesses |
| 334 | constexpr int ThreadsPerRowQ = RankQDim / kElemsPerAccess<DType>; |
| 335 | constexpr int ThreadsPerRowK = RankKDim / kElemsPerAccess<DType>; |
| 336 | // Number of warps dedicated to Q / K |
| 337 | constexpr int NumWarpQ = (ThreadsPerRowQ + MINIMAX_REDUCE_RMS_WARP_SIZE - 1) / |
| 338 | MINIMAX_REDUCE_RMS_WARP_SIZE; |
| 339 | constexpr int NumWarpK = (ThreadsPerRowK + MINIMAX_REDUCE_RMS_WARP_SIZE - 1) / |
| 340 | MINIMAX_REDUCE_RMS_WARP_SIZE; |
| 341 | |
| 342 | int tot_tokens = params.size_q / RankQDim; |
| 343 | int tot_groups = (tot_tokens + 3) / 4; // ceiling; last group may be partial |
| 344 | |
| 345 | // Memory strides for strided qkv tensors (elements -> float4-access units) |
| 346 | int access_stride_q = (params.stride_q > 0 ? params.stride_q : RankQDim) / |
| 347 | kElemsPerAccess<DType>; |
| 348 | int access_stride_k = (params.stride_k > 0 ? params.stride_k : RankKDim) / |
| 349 | kElemsPerAccess<DType>; |
| 350 | // Output strides: default to contiguous (hidden_dim / hidden_dim_k) |
| 351 | int access_stride_q_out = |
| 352 | (params.stride_q_out > 0 ? params.stride_q_out : params.hidden_dim) / |
| 353 | kElemsPerAccess<DType>; |
| 354 | int access_stride_k_out = |
| 355 | (params.stride_k_out > 0 ? params.stride_k_out : params.hidden_dim_k) / |
| 356 | kElemsPerAccess<DType>; |
| 357 | |
| 358 | #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) |
| 359 | namespace cg = cooperative_groups; |
| 360 | cg::cluster_group cluster = cg::this_cluster(); |
| 361 | cg::grid_group grid = cg::this_grid(); |
| 362 | int group_id = grid.cluster_rank(); |
| 363 | int access_id_in_token = cluster.thread_rank(); |
| 364 | int group_stride = grid.num_clusters(); |
| 365 | #else |
| 366 | int group_id = blockIdx.x; |
| 367 | int access_id_in_token = threadIdx.x; |
| 368 | int group_stride = gridDim.x; |
| 369 | #endif |
| 370 | |
| 371 | bool is_q = (access_id_in_token < NumWarpQ * MINIMAX_REDUCE_RMS_WARP_SIZE); |
| 372 | int k_thread_idx = |
| 373 | access_id_in_token - (NumWarpQ * MINIMAX_REDUCE_RMS_WARP_SIZE); |
| 374 | bool is_valid_q = (access_id_in_token < ThreadsPerRowQ); |
| 375 | bool is_valid_k = (k_thread_idx >= 0 && k_thread_idx < ThreadsPerRowK); |
| 376 | float4 clear_vec = get_neg_zero(); |
| 377 | |
| 378 | // Shared memory for two-level block reduction and scale broadcast |
| 379 | __shared__ float block_reduce_sum[4][MINIMAX_REDUCE_RMS_WARP_SIZE + 1]; |
| 380 | __shared__ float global_scale_q[4]; |
| 381 | __shared__ float global_scale_k[4]; |
| 382 | |
| 383 | LamportComm<NRanks> comm(params.workspace, params.rank); |
| 384 | |
| 385 | DType norm_weight[kElemsPerAccess<DType>]{}; |
| 386 | #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) |
| 387 | cudaGridDependencySynchronize(); |
| 388 | #endif |
| 389 | if (is_q) { |
| 390 | if (is_valid_q) { |
| 391 | *reinterpret_cast<typename ElemsPerAccess<DType>::vec_type*>( |
| 392 | norm_weight) = |
| 393 | reinterpret_cast<typename ElemsPerAccess<DType>::vec_type const*>( |
| 394 | params.rms_gamma)[access_id_in_token]; |
| 395 | } |
| 396 | } else { |
| 397 | if (is_valid_k) { |
| 398 | *reinterpret_cast<typename ElemsPerAccess<DType>::vec_type*>( |
| 399 | norm_weight) = |
| 400 | reinterpret_cast<typename ElemsPerAccess<DType>::vec_type const*>( |
| 401 | params.rms_gamma_k)[k_thread_idx]; |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | // Main loop: process one group of 4 tokens per iteration. |
| 406 | for (int g = group_id; g < tot_groups; g += group_stride) { |
| 407 | alignas(16) DType vals[4][kElemsPerAccess<DType>]{}; |
| 408 | float warp_sum_variance[4]{0.F, 0.F, 0.F, 0.F}; |
| 409 | |
| 410 | if (is_q) { |
| 411 | #pragma unroll |
| 412 | for (int row = 0; row < 4; ++row) { |
| 413 | int token_r = g * 4 + row; |
| 414 | if (token_r >= tot_tokens || !is_valid_q) { |
| 415 | continue; |
| 416 | } |
| 417 | int idx_r = token_r * access_stride_q + access_id_in_token; |
| 418 | *reinterpret_cast<float4*>(&vals[row][0]) = |
| 419 | reinterpret_cast<float4 const*>(params.allreduce_in)[idx_r]; |
| 420 | #pragma unroll |
| 421 | for (int i = 0; i < kElemsPerAccess<DType>; ++i) { |
| 422 | float x = static_cast<float>(vals[row][i]); |
| 423 | warp_sum_variance[row] += x * x; |
| 424 | } |
| 425 | } |
| 426 | } else { |
| 427 | #pragma unroll |
| 428 | for (int row = 0; row < 4; ++row) { |
| 429 | int token_r = g * 4 + row; |
| 430 | if (token_r >= tot_tokens || !is_valid_k) { |
| 431 | continue; |
| 432 | } |
| 433 | int idx_r = token_r * access_stride_k + k_thread_idx; |
| 434 | *reinterpret_cast<float4*>(&vals[row][0]) = |
| 435 | reinterpret_cast<float4 const*>(params.allreduce_in_k)[idx_r]; |
| 436 | #pragma unroll |
| 437 | for (int i = 0; i < kElemsPerAccess<DType>; ++i) { |
| 438 | float x = static_cast<float>(vals[row][i]); |
| 439 | warp_sum_variance[row] += x * x; |
| 440 | } |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | local_warp_reduce_sum_array<MINIMAX_REDUCE_RMS_WARP_SIZE, float, 4>( |
| 445 | warp_sum_variance); |
| 446 | // Warp lane 0 writes its warp's partial sum to shared memory |
| 447 | int lane = threadIdx.x & (MINIMAX_REDUCE_RMS_WARP_SIZE - 1); |
| 448 | if (lane == 0) { |
| 449 | #pragma unroll |
| 450 | for (int t = 0; t < 4; ++t) { |
| 451 | block_reduce_sum[t][threadIdx.x / MINIMAX_REDUCE_RMS_WARP_SIZE] = |
| 452 | warp_sum_variance[t]; |
| 453 | } |
| 454 | } |
| 455 | __syncthreads(); |
| 456 | |
| 457 | int tid = threadIdx.x; |
| 458 | |
| 459 | if (tid < MINIMAX_REDUCE_RMS_WARP_SIZE) { |
| 460 | constexpr int kNumWarpQPow2 = |
| 461 | (next_pow2(NumWarpQ) > NRanks) ? next_pow2(NumWarpQ) : NRanks; |
| 462 | float local_sum[4]; |
| 463 | #pragma unroll |
| 464 | for (int t = 0; t < 4; ++t) { |
| 465 | local_sum[t] = (tid < NumWarpQ) ? block_reduce_sum[t][tid] : 0.F; |
| 466 | } |
| 467 | // After this, all kNumWarpQPow2 lanes (including tid 0..NRanks-1) have |
| 468 | // the total Q sum-of-squares for all 4 tokens. |
| 469 | local_warp_reduce_sum_array<kNumWarpQPow2, float, 4>(local_sum); |
| 470 | |
| 471 | if (tid < NRanks) { |
| 472 | #pragma unroll |
| 473 | for (int t = 0; t < 4; ++t) { |
| 474 | if (is_neg_zero(local_sum[t])) { |
| 475 | local_sum[t] = 0.F; |
| 476 | } |
| 477 | } |
| 478 | // Parallel push: thread tid writes this rank's Q sum to rank tid's buf |
| 479 | reinterpret_cast<float4*>( |
| 480 | comm.data_bufs[tid])[(params.rank * tot_groups * 2) + (2 * g)] = |
| 481 | *reinterpret_cast<float4*>(local_sum); |
| 482 | |
| 483 | // Parallel pull: thread tid reads rank tid's contribution from |
| 484 | // this rank's (params.rank's) buffer |
| 485 | bool done = false; |
| 486 | float4 var_all_ranks; |
| 487 | while (!done) { |
| 488 | done = true; |
| 489 | var_all_ranks = ld_global_volatile(&reinterpret_cast<float4*>( |
| 490 | comm.data_bufs[params.rank])[(tid * tot_groups * 2) + (2 * g)]); |
| 491 | done &= !is_neg_zero(var_all_ranks); |
| 492 | } |
| 493 | |
| 494 | // Warp-level allreduce: each of the NRanks threads holds one rank's |
| 495 | // partial sum; after this all NRanks threads have the global total. |
| 496 | constexpr uint32_t kQActiveMask = (1u << NRanks) - 1u; |
| 497 | local_warp_reduce_sum_array<NRanks, float, 4>( |
| 498 | reinterpret_cast<float*>(&var_all_ranks), kQActiveMask); |
| 499 | |
| 500 | // Thread 0 computes rsqrt with compile-time Dim and writes to smem |
| 501 | if (tid == 0) { |
| 502 | *reinterpret_cast<float4*>(global_scale_q) = |
| 503 | rms_rsqrt<OriginQDim>(var_all_ranks, params.rms_eps); |
| 504 | } |
| 505 | } |
| 506 | } else if (tid >= MINIMAX_REDUCE_RMS_WARP_SIZE * NumWarpQ && |
| 507 | tid < MINIMAX_REDUCE_RMS_WARP_SIZE * (NumWarpQ + 1)) { |
| 508 | // --- K leader warp --- |
| 509 | constexpr int kNumWarpKPow2 = |
| 510 | (next_pow2(NumWarpK) > NRanks) ? next_pow2(NumWarpK) : NRanks; |
| 511 | float local_sum[4]; |
| 512 | #pragma unroll |
| 513 | for (int t = 0; t < 4; ++t) { |
| 514 | local_sum[t] = (k_thread_idx < NumWarpK) |
| 515 | ? block_reduce_sum[t][NumWarpQ + k_thread_idx] |
| 516 | : 0.F; |
| 517 | } |
| 518 | local_warp_reduce_sum_array<kNumWarpKPow2, float, 4>(local_sum); |
| 519 | |
| 520 | if (k_thread_idx < NRanks) { |
| 521 | #pragma unroll |
| 522 | for (int t = 0; t < 4; ++t) { |
| 523 | if (is_neg_zero(local_sum[t])) { |
| 524 | local_sum[t] = 0.F; |
| 525 | } |
| 526 | } |
| 527 | reinterpret_cast<float4*>( |
| 528 | comm.data_bufs[k_thread_idx])[(params.rank * tot_groups * 2) + |
| 529 | (2 * g + 1)] = |
| 530 | *reinterpret_cast<float4*>(local_sum); |
| 531 | |
| 532 | bool done = false; |
| 533 | float4 var_all_ranks; |
| 534 | while (!done) { |
| 535 | done = true; |
| 536 | var_all_ranks = ld_global_volatile(&reinterpret_cast<float4*>( |
| 537 | comm.data_bufs[params.rank])[(k_thread_idx * tot_groups * 2) + |
| 538 | (2 * g + 1)]); |
| 539 | done &= !is_neg_zero(var_all_ranks); |
| 540 | } |
| 541 | |
| 542 | constexpr uint32_t kKActiveMask = (1u << NRanks) - 1u; |
| 543 | local_warp_reduce_sum_array<NRanks, float, 4>( |
| 544 | reinterpret_cast<float*>(&var_all_ranks), kKActiveMask); |
| 545 | |
| 546 | if (k_thread_idx == 0) { |
| 547 | *reinterpret_cast<float4*>(global_scale_k) = |
| 548 | rms_rsqrt<OriginKDim>(var_all_ranks, params.rms_eps); |
| 549 | } |
| 550 | } |
| 551 | } |
| 552 | __syncthreads(); |
| 553 | |
| 554 | if (is_q) { |
| 555 | #pragma unroll |
| 556 | for (int t = 0; t < 4; ++t) { |
| 557 | warp_sum_variance[t] = global_scale_q[t]; |
| 558 | } |
| 559 | #pragma unroll |
| 560 | for (int r = 0; r < 4; ++r) { |
| 561 | #pragma unroll |
| 562 | for (int i = 0; i < kElemsPerAccess<DType>; ++i) { |
| 563 | vals[r][i] = static_cast<DType>(static_cast<float>(vals[r][i]) * |
| 564 | warp_sum_variance[r] * |
| 565 | static_cast<float>(norm_weight[i])); |
| 566 | } |
| 567 | int token_r = g * 4 + r; |
| 568 | if (token_r >= tot_tokens || !is_valid_q) { |
| 569 | continue; |
| 570 | } |
| 571 | int idx_out = token_r * access_stride_q_out + access_id_in_token; |
| 572 | reinterpret_cast<float4*>(params.rms_norm_out)[idx_out] = |
| 573 | *reinterpret_cast<float4*>(&vals[r][0]); |
| 574 | } |
| 575 | } else { |
| 576 | #pragma unroll |
| 577 | for (int t = 0; t < 4; ++t) { |
| 578 | warp_sum_variance[t] = global_scale_k[t]; |
| 579 | } |
| 580 | #pragma unroll |
| 581 | for (int r = 0; r < 4; ++r) { |
| 582 | #pragma unroll |
| 583 | for (int i = 0; i < kElemsPerAccess<DType>; ++i) { |
| 584 | vals[r][i] = static_cast<DType>(static_cast<float>(vals[r][i]) * |
| 585 | warp_sum_variance[r] * |
| 586 | static_cast<float>(norm_weight[i])); |
| 587 | } |
| 588 | int token_r = g * 4 + r; |
| 589 | if (token_r >= tot_tokens || !is_valid_k) { |
| 590 | continue; |
| 591 | } |
| 592 | int idx_out = token_r * access_stride_k_out + k_thread_idx; |
| 593 | reinterpret_cast<float4*>(params.rms_norm_out_k)[idx_out] = |
| 594 | *reinterpret_cast<float4*>(&vals[r][0]); |
| 595 | } |
| 596 | } |
| 597 | } // end group loop |
| 598 | #if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) |
| 599 | cudaTriggerProgrammaticLaunchCompletion(); |
| 600 | #endif |
| 601 | |
| 602 | int clear_access = static_cast<int>(comm.clear_size / kElemsPerAccess<DType>); |
| 603 | int clear_stride = group_stride * blockDim.x; |
| 604 | for (int idx = group_id * blockDim.x + threadIdx.x; idx < clear_access; |
| 605 | idx += clear_stride) { |
| 606 | reinterpret_cast<float4*>(comm.clear_buf)[idx] = clear_vec; |
| 607 | } |
| 608 | |
| 609 | comm.update(static_cast<int64_t>(2) * tot_groups * kElemsPerAccess<DType> * |
| 610 | NRanks); |
| 611 | } |
| 612 | |
| 613 | int get_sm_count() { |
| 614 | static int sm_count = 0; |
| 615 | if (sm_count == 0) { |
| 616 | int device_id; |
| 617 | STD_CUDA_CHECK(cudaGetDevice(&device_id)); |
| 618 | cudaDeviceProp device_prop; |
| 619 | cudaGetDeviceProperties(&device_prop, device_id); |
| 620 | sm_count = device_prop.multiProcessorCount; |
| 621 | } |
| 622 | return sm_count; |
| 623 | } |
| 624 | |
| 625 | inline int getSMVersion(bool queryRealSmArch = false) { |
| 626 | int device{-1}; |
| 627 | STD_CUDA_CHECK(cudaGetDevice(&device)); |
| 628 | int sm_major = 0; |
| 629 | int sm_minor = 0; |
| 630 | STD_CUDA_CHECK(cudaDeviceGetAttribute( |
| 631 | &sm_major, cudaDevAttrComputeCapabilityMajor, device)); |
| 632 | STD_CUDA_CHECK(cudaDeviceGetAttribute( |
| 633 | &sm_minor, cudaDevAttrComputeCapabilityMinor, device)); |
| 634 | int sm = sm_major * 10 + sm_minor; |
| 635 | if (sm == 121 && !queryRealSmArch) { |
| 636 | return 120; |
| 637 | } |
| 638 | return sm; |
| 639 | } |
| 640 | |
| 641 | template <typename KernelFunc> |
| 642 | int get_max_active_blocks(KernelFunc kernel, int block_size, |
| 643 | int dynamic_smem = 0) { |
| 644 | int max_active = 0; |
| 645 | STD_CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor( |
| 646 | &max_active, kernel, block_size, dynamic_smem)); |
| 647 | return std::max(max_active, 1); |
| 648 | } |
| 649 | |
| 650 | template <typename DType, int NRanks> |
| 651 | void minimax_reduce_rms_kernel_launcher(MiniMaxReduceRMSParams const& params) { |
| 652 | static int SM = getSMVersion(); |
| 653 | int token_num = params.size_q / params.hidden_dim; |
| 654 | int sm_count = get_sm_count(); |
| 655 | int cluster_size = 1; |
| 656 | int cluster_num = token_num; |
| 657 | int threads_per_token = params.hidden_dim / kElemsPerAccess<DType>; |
| 658 | int block_size = threads_per_token; |
| 659 | |
| 660 | int max_blocks_per_sm = get_max_active_blocks( |
| 661 | minimax_reduce_rms_kernel_lamport<DType, NRanks>, block_size); |
| 662 | int max_grid = max_blocks_per_sm * sm_count; |
| 663 | |
| 664 | int grid_size = |
| 665 | (std::min(max_grid, cluster_num * cluster_size) / cluster_size) * |
| 666 | cluster_size; |
| 667 | |
| 668 | cudaLaunchConfig_t cfg; |
| 669 | cfg.gridDim = grid_size; |
| 670 | cfg.blockDim = block_size; |
| 671 | cfg.dynamicSmemBytes = 0; |
| 672 | cfg.stream = params.stream; |
| 673 | |
| 674 | cudaLaunchAttribute attribute[2]; |
| 675 | attribute[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; |
| 676 | attribute[0].val.programmaticStreamSerializationAllowed = 1; |
| 677 | attribute[1].id = cudaLaunchAttributeClusterDimension; |
| 678 | attribute[1].val.clusterDim.x = cluster_size; |
| 679 | attribute[1].val.clusterDim.y = 1; |
| 680 | attribute[1].val.clusterDim.z = 1; |
| 681 | cfg.attrs = attribute; |
| 682 | cfg.numAttrs = SM >= 90 ? 2 : 0; |
| 683 | |
| 684 | STD_CUDA_CHECK(cudaLaunchKernelEx( |
| 685 | &cfg, minimax_reduce_rms_kernel_lamport<DType, NRanks>, params)); |
| 686 | } |
| 687 | |
| 688 | template <typename DType, int NRanks, int OriginQDim, int OriginKDim> |
| 689 | void minimax_reduce_rms_kernel_launcher_float4( |
| 690 | MiniMaxReduceRMSParams const& params) { |
| 691 | STD_TORCH_CHECK(params.size_q % params.hidden_dim == 0); |
| 692 | STD_TORCH_CHECK(params.hidden_dim % kElemsPerAccess<DType> == 0); |
| 693 | if (params.stride_q > 0) { |
| 694 | STD_TORCH_CHECK(params.stride_q % kElemsPerAccess<DType> == 0); |
| 695 | } |
| 696 | STD_TORCH_CHECK(params.allreduce_in_k != nullptr, |
| 697 | "float4 QK kernel requires K input"); |
| 698 | STD_TORCH_CHECK(params.hidden_dim >= params.hidden_dim_k); |
| 699 | STD_TORCH_CHECK(params.size_k % params.hidden_dim_k == 0); |
| 700 | STD_TORCH_CHECK(params.hidden_dim_k % kElemsPerAccess<DType> == 0); |
| 701 | STD_TORCH_CHECK(params.size_q / params.hidden_dim == |
| 702 | params.size_k / params.hidden_dim_k); |
| 703 | if (params.stride_k > 0) { |
| 704 | STD_TORCH_CHECK(params.stride_k % kElemsPerAccess<DType> == 0); |
| 705 | } |
| 706 | |
| 707 | int token_num = params.size_q / params.hidden_dim; |
| 708 | int tot_groups = (token_num + 3) / 4; |
| 709 | if (tot_groups == 0) { |
| 710 | return; |
| 711 | } |
| 712 | |
| 713 | static int SM = getSMVersion(); |
| 714 | int sm_count = get_sm_count(); |
| 715 | int cluster_size = 1; |
| 716 | int cluster_num = tot_groups; |
| 717 | |
| 718 | int access_per_row_q = params.hidden_dim / kElemsPerAccess<DType>; |
| 719 | int access_per_row_k = params.hidden_dim_k / kElemsPerAccess<DType>; |
| 720 | |
| 721 | // Round each section up to a warp boundary |
| 722 | auto divUp = [](int a, int b) { return (a + b - 1) / b * b; }; |
| 723 | int block_size = divUp(access_per_row_q, MINIMAX_REDUCE_RMS_WARP_SIZE) + |
| 724 | divUp(access_per_row_k, MINIMAX_REDUCE_RMS_WARP_SIZE); |
| 725 | |
| 726 | auto kfn = |
| 727 | minimax_reduce_qk_rms_kernel_lamport_float4<DType, NRanks, OriginQDim, |
| 728 | OriginKDim>; |
| 729 | |
| 730 | int max_blocks_per_sm = get_max_active_blocks(kfn, block_size); |
| 731 | int max_grid = max_blocks_per_sm * sm_count; |
| 732 | int grid_size = |
| 733 | (std::min(max_grid, cluster_num * cluster_size) / cluster_size) * |
| 734 | cluster_size; |
| 735 | |
| 736 | cudaLaunchConfig_t cfg; |
| 737 | cfg.gridDim = grid_size; |
| 738 | cfg.blockDim = block_size; |
| 739 | cfg.dynamicSmemBytes = 0; |
| 740 | cfg.stream = params.stream; |
| 741 | |
| 742 | cudaLaunchAttribute attribute[2]; |
| 743 | attribute[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; |
| 744 | attribute[0].val.programmaticStreamSerializationAllowed = 1; |
| 745 | attribute[1].id = cudaLaunchAttributeClusterDimension; |
| 746 | attribute[1].val.clusterDim.x = cluster_size; |
| 747 | attribute[1].val.clusterDim.y = 1; |
| 748 | attribute[1].val.clusterDim.z = 1; |
| 749 | cfg.attrs = attribute; |
| 750 | cfg.numAttrs = SM >= 90 ? 2 : 0; |
| 751 | |
| 752 | STD_CUDA_CHECK(cudaLaunchKernelEx(&cfg, kfn, params)); |
| 753 | } |
| 754 | |
| 755 | template <int NRanks> |
| 756 | void dispatch_dtype(MiniMaxReduceRMSParams const& params) { |
| 757 | // Use the optimized QK float4 kernel when: |
| 758 | // - K input is present, AND |
| 759 | // - the full (NRanks * per-rank) dimensions match the MiniMax M2 shape. |
| 760 | // Otherwise fall back to the scalar kernel. |
| 761 | bool use_float4 = (params.allreduce_in_k != nullptr) && |
| 762 | (params.hidden_dim * params.nranks == 6144) && |
| 763 | (params.hidden_dim_k * params.nranks == 1024); |
| 764 | |
| 765 | if (params.dtype == torch::headeronly::ScalarType::Half) { |
| 766 | if (use_float4) { |
| 767 | minimax_reduce_rms_kernel_launcher_float4<half, NRanks, 6144, 1024>( |
| 768 | params); |
| 769 | } else { |
| 770 | minimax_reduce_rms_kernel_launcher<half, NRanks>(params); |
| 771 | } |
| 772 | } else if (params.dtype == torch::headeronly::ScalarType::BFloat16) { |
| 773 | if (use_float4) { |
| 774 | minimax_reduce_rms_kernel_launcher_float4<__nv_bfloat16, NRanks, 6144, |
| 775 | 1024>(params); |
| 776 | } else { |
| 777 | minimax_reduce_rms_kernel_launcher<__nv_bfloat16, NRanks>(params); |
| 778 | } |
| 779 | } else if (params.dtype == torch::headeronly::ScalarType::Float) { |
| 780 | if (use_float4) { |
| 781 | minimax_reduce_rms_kernel_launcher_float4<float, NRanks, 6144, 1024>( |
| 782 | params); |
| 783 | } else { |
| 784 | minimax_reduce_rms_kernel_launcher<float, NRanks>(params); |
| 785 | } |
| 786 | } else { |
| 787 | STD_TORCH_CHECK(false, "Unsupported data type for minimax_reduce_rms_op"); |
| 788 | } |
| 789 | } |
| 790 | |
| 791 | void minimax_reduce_rms_op(MiniMaxReduceRMSParams const& params) { |
| 792 | if (params.nranks == 2) { |
| 793 | dispatch_dtype<2>(params); |
| 794 | } else if (params.nranks == 4) { |
| 795 | dispatch_dtype<4>(params); |
| 796 | } else if (params.nranks == 8) { |
| 797 | dispatch_dtype<8>(params); |
| 798 | } else if (params.nranks == 16) { |
| 799 | dispatch_dtype<16>(params); |
| 800 | } else { |
| 801 | STD_TORCH_CHECK(false, "minimax_reduce_rms_op: unsupported ranks number!"); |
| 802 | } |
| 803 | } |
| 804 | } // namespace tensorrt_llm |
| 805 | } // namespace vllm |
| 806 | |
| 807 | std::tuple<torch::stable::Tensor, torch::stable::Tensor> |
| 808 | minimax_allreduce_rms_qk(torch::stable::Tensor qkv, |
| 809 | torch::stable::Tensor const& norm_weight_q, |
| 810 | torch::stable::Tensor const& norm_weight_k, |
| 811 | torch::stable::Tensor workspace, int64_t const q_size, |
| 812 | int64_t const kv_size, int64_t const rank, |
| 813 | int64_t const nranks, double const eps) { |
| 814 | STD_TORCH_CHECK(qkv.dim() == 2, "minimax_allreduce_rms_qk: qkv must be 2D"); |
| 815 | STD_TORCH_CHECK(qkv.is_contiguous(), |
| 816 | "minimax_allreduce_rms_qk: qkv must be contiguous"); |
| 817 | int64_t qkv_dim = qkv.size(-1); |
| 818 | STD_TORCH_CHECK(qkv_dim == q_size + 2 * kv_size, |
| 819 | "minimax_allreduce_rms_qk: qkv last dim must equal " |
| 820 | "q_size + 2 * kv_size"); |
| 821 | STD_TORCH_CHECK(rank < nranks, |
| 822 | "minimax_allreduce_rms_qk: rank must be less than nranks"); |
| 823 | |
| 824 | const torch::stable::accelerator::DeviceGuard device_guard( |
| 825 | qkv.get_device_index()); |
| 826 | |
| 827 | int64_t num_tokens = qkv.size(0); |
| 828 | int elem_bytes = qkv.element_size(); |
| 829 | |
| 830 | torch::stable::Tensor q_out = |
| 831 | torch::stable::new_empty(qkv, {num_tokens, q_size}, qkv.scalar_type()); |
| 832 | torch::stable::Tensor k_out = |
| 833 | torch::stable::new_empty(qkv, {num_tokens, kv_size}, qkv.scalar_type()); |
| 834 | |
| 835 | auto params = vllm::tensorrt_llm::MiniMaxReduceRMSParams(); |
| 836 | params.nranks = static_cast<int>(nranks); |
| 837 | params.rank = static_cast<int>(rank); |
| 838 | params.dtype = qkv.scalar_type(); |
| 839 | params.size_q = static_cast<int>(num_tokens * q_size); |
| 840 | params.hidden_dim = static_cast<int>(q_size); |
| 841 | params.size_k = static_cast<int>(num_tokens * kv_size); |
| 842 | params.hidden_dim_k = static_cast<int>(kv_size); |
| 843 | params.stride_q = static_cast<int>(qkv_dim); |
| 844 | params.stride_k = static_cast<int>(qkv_dim); |
| 845 | params.stride_q_out = 0; // q_out is contiguous; kernel uses hidden_dim |
| 846 | params.stride_k_out = 0; // k_out is contiguous; kernel uses hidden_dim_k |
| 847 | params.workspace = reinterpret_cast<void**>(workspace.mutable_data_ptr()); |
| 848 | |
| 849 | uint8_t* base = |
| 850 | const_cast<uint8_t*>(static_cast<const uint8_t*>(qkv.const_data_ptr())); |
| 851 | params.allreduce_in = base; |
| 852 | params.allreduce_in_k = base + q_size * elem_bytes; |
| 853 | params.rms_gamma = const_cast<void*>(norm_weight_q.const_data_ptr()); |
| 854 | params.rms_gamma_k = const_cast<void*>(norm_weight_k.const_data_ptr()); |
| 855 | params.rms_eps = static_cast<float>(eps); |
| 856 | params.stream = get_current_cuda_stream(qkv.get_device_index()); |
| 857 | |
| 858 | params.rms_norm_out = q_out.mutable_data_ptr(); |
| 859 | params.rms_norm_out_k = k_out.mutable_data_ptr(); |
| 860 | |
| 861 | vllm::tensorrt_llm::minimax_reduce_rms_op(params); |
| 862 | return {q_out, k_out}; |
| 863 | } |