@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a928ded995
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Kernel] Split Marlin MoE kernels into multiple files (#8661)

Co-authored-by: mgoin <michael@neuralmagic.com>

ElizaWszola committed Sep 24, 2024 at 18:31 UTC a928ded99519f803d4cf6389df6acc707239a5cc
7 files changed +1552 -1427
CMakeLists.txt
+5
@@ -316,6 +316,11 @@ set(VLLM_MOE_EXT_SRC
316
317 if(VLLM_GPU_LANG STREQUAL "CUDA")
318 list(APPEND VLLM_MOE_EXT_SRC
319 + "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
320 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h"
321 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
322 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
323 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
324 "csrc/moe/marlin_moe_ops.cu")
325 endif()
326
csrc/moe/marlin_kernels/marlin_moe_kernel.h new
+1425
@@ -0,0 +1,1425 @@
1 +#pragma once
2 +
3 +#include <torch/all.h>
4 +
5 +#include <ATen/cuda/CUDAContext.h>
6 +#include <c10/cuda/CUDAGuard.h>
7 +#include <cuda.h>
8 +#include <cuda_fp16.h>
9 +#include <cuda_runtime.h>
10 +
11 +#include <iostream>
12 +
13 +#include "core/scalar_type.hpp"
14 +
15 +namespace marlin_moe {
16 +
17 +constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
18 +
19 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
20 +
21 +// Instances of `Vec` are used to organize groups of >>registers<<, as needed
22 +// for instance as inputs to tensor core operations. Consequently, all
23 +// corresponding index accesses must be compile-time constants, which is why we
24 +// extensively use `#pragma unroll` throughout the kernel code to guarantee
25 +// this.
26 +template <typename T, int n>
27 +struct Vec {
28 + T elems[n];
29 + __device__ T& operator[](int i) { return elems[i]; }
30 +};
31 +
32 +using I4 = Vec<int, 4>;
33 +
34 +// Matrix fragments for tensor core instructions; their precise layout is
35 +// documented here:
36 +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
37 +using FragA = Vec<half2, 4>;
38 +using FragB = Vec<half2, 2>;
39 +using FragC = Vec<float, 4>;
40 +using FragS = Vec<half2, 1>; // quantization scales
41 +
42 +// Predicated asynchronous global->shared copy; used for inputs A where we apply
43 +// predication to handle batchsizes that are not multiples of 16.
44 +__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
45 + bool pred = true) {
46 + const int BYTES = 16;
47 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
48 + asm volatile(
49 + "{\n"
50 + " .reg .pred p;\n"
51 + " setp.ne.b32 p, %0, 0;\n"
52 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
53 + "}\n" ::"r"((int)pred),
54 + "r"(smem), "l"(glob_ptr), "n"(BYTES));
55 +}
56 +
57 +// Asynchronous global->shared copy
58 +__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
59 + const int BYTES = 16;
60 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
61 + asm volatile(
62 + "{\n"
63 + " cp.async.cg.shared.global [%0], [%1], %2;\n"
64 + "}\n" ::"r"(smem),
65 + "l"(glob_ptr), "n"(BYTES));
66 +}
67 +
68 +// Async copy fence.
69 +__device__ inline void cp_async_fence() {
70 + asm volatile("cp.async.commit_group;\n" ::);
71 +}
72 +
73 +// Wait until at most `n` async copy stages are still pending.
74 +template <int n>
75 +__device__ inline void cp_async_wait() {
76 + asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
77 +}
78 +
79 +// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
80 +// output/accumulation.
81 +__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
82 + FragC& frag_c) {
83 + const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
84 + const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
85 + float* c = reinterpret_cast<float*>(&frag_c);
86 + asm volatile(
87 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
88 + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
89 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
90 + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
91 + "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
92 +}
93 +
94 +// Instruction for loading a full 16x16 matrix fragment of operand A from shared
95 +// memory, directly in tensor core layout.
96 +__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
97 + uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
98 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
99 + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
100 + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
101 + : "r"(smem));
102 +}
103 +
104 +// Lookup-table based 3-input logical operation; explicitly used for
105 +// dequantization as the compiler does not seem to automatically recognize it in
106 +// all cases.
107 +template <int lut>
108 +__device__ inline int lop3(int a, int b, int c) {
109 + int res;
110 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
111 + : "=r"(res)
112 + : "r"(a), "r"(b), "r"(c), "n"(lut));
113 + return res;
114 +}
115 +
116 +// Constructs destination register by taking bytes from 2 sources (based on
117 +// mask)
118 +template <int start_byte, int mask>
119 +__device__ inline uint32_t prmt(uint32_t a) {
120 + uint32_t res;
121 + asm volatile("prmt.b32 %0, %1, %2, %3;\n"
122 + : "=r"(res)
123 + : "r"(a), "n"(start_byte), "n"(mask));
124 + return res;
125 +}
126 +
127 +template <vllm::ScalarTypeId w_type_id>
128 +__device__ inline FragB dequant(int q);
129 +
130 +// Efficiently dequantize 4bit values packed in an int32 value into a full
131 +// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
132 +// with some small changes:
133 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
134 +template <>
135 +__device__ inline FragB dequant<vllm::kU4B8.id()>(int q) {
136 + const int LO = 0x000f000f;
137 + const int HI = 0x00f000f0;
138 + const int EX = 0x64006400;
139 + // Guarantee that the `(a & b) | c` operations are LOP3s.
140 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
141 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
142 + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
143 + // directly into `SUB` and `ADD`.
144 + const int SUB = 0x64086408;
145 + const int MUL = 0x2c002c00;
146 + const int ADD = 0xd480d480;
147 + FragB frag_b;
148 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
149 + *reinterpret_cast<const half2*>(&SUB));
150 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
151 + *reinterpret_cast<const half2*>(&MUL),
152 + *reinterpret_cast<const half2*>(&ADD));
153 + return frag_b;
154 +}
155 +
156 +// Fast Int8ToFp16: Efficiently dequantize 8bit int values to fp16
157 +// Reference:
158 +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
159 +template <>
160 +__device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
161 + static constexpr uint32_t mask_for_elt_01 = 0x5250;
162 + static constexpr uint32_t mask_for_elt_23 = 0x5351;
163 + static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
164 +
165 + uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
166 + uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
167 +
168 + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
169 +
170 + FragB frag_b;
171 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
172 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
173 + frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
174 + *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
175 + return frag_b;
176 +}
177 +
178 +// Multiply dequantized values by the corresponding quantization scale; used
179 +// only for grouped quantization.
180 +__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
181 + half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
182 + frag_b[0] = __hmul2(frag_b[0], s);
183 + frag_b[1] = __hmul2(frag_b[1], s);
184 +}
185 +
186 +// Given 2 floats multiply by 2 scales (halves)
187 +__device__ inline void scale_float(float* c, FragS& s) {
188 + __half* s_ptr = reinterpret_cast<__half*>(&s);
189 + c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
190 + c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
191 +}
192 +
193 +// Same as above, but for act_order (each K is multiplied individually)
194 +__device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
195 + FragS& frag_s_3, FragS& frag_s_4, int i) {
196 + __half2 s_val_1_2;
197 + s_val_1_2.x = reinterpret_cast<__half*>(&frag_s_1)[i];
198 + s_val_1_2.y = reinterpret_cast<__half*>(&frag_s_2)[i];
199 +
200 + __half2 s_val_3_4;
201 + s_val_3_4.x = reinterpret_cast<__half*>(&frag_s_3)[i];
202 + s_val_3_4.y = reinterpret_cast<__half*>(&frag_s_4)[i];
203 +
204 + frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
205 + frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
206 +}
207 +
208 +// Wait until barrier reaches `count`, then lock for current threadblock.
209 +__device__ inline void barrier_acquire(int* lock, int count) {
210 + if (threadIdx.x == 0) {
211 + int state = -1;
212 + do
213 + // Guarantee that subsequent writes by this threadblock will be visible
214 + // globally.
215 + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
216 + : "=r"(state)
217 + : "l"(lock));
218 + while (state != count);
219 + }
220 + __syncthreads();
221 +}
222 +
223 +// Release barrier and increment visitation count.
224 +__device__ inline void barrier_release(int* lock, bool reset = false) {
225 + __syncthreads();
226 + if (threadIdx.x == 0) {
227 + if (reset) {
228 + lock[0] = 0;
229 + return;
230 + }
231 + int val = 1;
232 + // Make sure that all writes since acquiring this barrier are visible
233 + // globally, while releasing the barrier.
234 + asm volatile("fence.acq_rel.gpu;\n");
235 + asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
236 + :
237 + : "l"(lock), "r"(val));
238 + }
239 +}
240 +
241 +template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
242 + const int threads, // number of threads in a threadblock
243 + const int thread_m_blocks, // number of 16x16 blocks in the m
244 + // dimension (batchsize) of the
245 + // threadblock
246 + const int thread_n_blocks, // same for n dimension (output)
247 + const int thread_k_blocks, // same for k dimension (reduction)
248 + const int stages, // number of stages for the async global->shared
249 + // fetch pipeline
250 + const bool has_act_order, // whether act_order is enabled
251 + const int group_blocks = -1 // number of consecutive 16x16 blocks
252 + // with a separate quantization scale
253 + >
254 +__device__ inline void MarlinMoESingle(
255 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
256 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
257 + int4* __restrict__ C, // fp16 output buffer of shape mxn
258 + const int* __restrict__ sorted_ids, // int32 sorted ids of experts
259 + const float* __restrict__ topk_weights, // float topk weights
260 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
261 + // (k/groupsize)xn
262 + const int* __restrict__ g_idx, // int32 group indices of shape k
263 + const int* __restrict__ expert_offsets,
264 + int num_groups, // number of scale groups per output channel
265 + int expert_idx, // idx of current expert
266 + int num_experts, // number of experts
267 + int topk, // topk parameter of moe
268 + int prob_m, // batch dimension m
269 + int prob_n, // output dimension n
270 + int prob_k, // reduction dimension k
271 + int tot_m, // total number of rows in A and C
272 + int* locks, // extra global storage for barrier synchronization
273 + bool replicate_input, // do we use the same input for each expert?
274 + bool apply_weights, // apply weights to output
275 + int current_m_block // current m block to start kernel computation from
276 +) {
277 + static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
278 + constexpr int pack_factor = 32 / w_type.size_bits();
279 +
280 + // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
281 + // better partitioning with less reductions
282 + int parallel = 1;
283 + if (prob_m > 16 * thread_m_blocks) {
284 + parallel = prob_m / (16 * thread_m_blocks);
285 + prob_m = 16 * thread_m_blocks;
286 + }
287 +
288 + int k_tiles = prob_k / 16 / thread_k_blocks;
289 + int n_tiles = prob_n / 16 / thread_n_blocks;
290 + int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
291 +
292 + if constexpr (!has_act_order && group_blocks != -1) {
293 + if (group_blocks >= thread_k_blocks) {
294 + // Ensure that the number of tiles in each stripe is a multiple of the
295 + // groupsize; this avoids an annoying special case where a stripe starts
296 + // in the middle of group.
297 + iters = (group_blocks / thread_k_blocks) *
298 + ceildiv(iters, (group_blocks / thread_k_blocks));
299 + }
300 + }
301 +
302 + int slice_row = (iters * blockIdx.x) % k_tiles;
303 + int slice_col_par = (iters * blockIdx.x) / k_tiles;
304 + int slice_col = slice_col_par;
305 + int slice_iters; // number of threadblock tiles in the current slice
306 + int slice_count =
307 + 0; // total number of active threadblocks in the current slice
308 + int slice_idx; // index of threadblock in current slice; numbered bottom to
309 + // top
310 +
311 + // We can easily implement parallel problem execution by just remapping
312 + // indices and advancing global pointers
313 + if (slice_col_par >= n_tiles) {
314 + locks += (slice_col_par / n_tiles) * n_tiles;
315 + slice_col = slice_col_par % n_tiles;
316 + sorted_ids += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
317 + }
318 +
319 + // Compute all information about the current slice which is required for
320 + // synchronization.
321 + auto init_slice = [&]() {
322 + slice_iters =
323 + iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
324 + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
325 + if (slice_iters == 0) return;
326 + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
327 + slice_count = 1;
328 + slice_idx = 0;
329 + int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
330 + if (col_first <= k_tiles * (slice_col_par + 1)) {
331 + int col_off = col_first - k_tiles * slice_col_par;
332 + slice_count = ceildiv(k_tiles - col_off, iters);
333 + if (col_off > 0) slice_count++;
334 + int delta_first = iters * blockIdx.x - col_first;
335 + if (delta_first < 0 || (col_off == 0 && delta_first == 0))
336 + slice_idx = slice_count - 1;
337 + else {
338 + slice_idx = slice_count - 1 - delta_first / iters;
339 + if (col_off > 0) slice_idx--;
340 + }
341 + }
342 + if (slice_col == n_tiles) {
343 + sorted_ids += 16 * thread_m_blocks;
344 + locks += n_tiles;
345 + slice_col = 0;
346 + }
347 + };
348 + init_slice();
349 +
350 + // A sizes/strides
351 +
352 + // stride of the A matrix in global memory
353 + int a_gl_stride = prob_k / 8;
354 + // stride of an A matrix tile in shared memory
355 + constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
356 + // delta between subsequent A tiles in global memory
357 + constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
358 + // between subsequent accesses within a tile
359 + int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
360 + // between shared memory writes
361 + constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
362 + // between shared memory tile reads
363 + constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
364 + // within a shared memory tile
365 + constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
366 + // overall size of a tile
367 + constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
368 + // number of shared write iterations for a tile
369 + constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
370 +
371 + // B sizes/strides
372 + int b_gl_stride = 16 * prob_n / (pack_factor * 4);
373 + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
374 + constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
375 + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
376 +
377 + int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
378 + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
379 + constexpr int b_sh_wr_delta = threads * b_thread_vecs;
380 + constexpr int b_sh_rd_delta = threads * b_thread_vecs;
381 + constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
382 + constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
383 +
384 + // Scale sizes/strides without act_order
385 + int s_gl_stride = prob_n / 8;
386 + constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
387 + constexpr int s_tb_groups =
388 + !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
389 + ? thread_k_blocks / group_blocks
390 + : 1;
391 + constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
392 + int s_gl_rd_delta = s_gl_stride;
393 + // Scale size/strides with act_order
394 + constexpr int tb_k = 16 * thread_k_blocks;
395 + constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
396 + // constexpr int act_s_row_stride = 1;
397 + // int act_s_col_stride = act_s_row_stride * num_groups;
398 + int act_s_col_stride = 1;
399 + int act_s_col_warp_stride = act_s_col_stride * 8;
400 + int tb_n_warps = thread_n_blocks / 4;
401 + int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
402 +
403 + constexpr int sorted_sh_stride = threads;
404 + constexpr int sorted_gl_stride = threads;
405 +
406 + // Global A read index of current thread.
407 + int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
408 + (threadIdx.x % a_gl_rd_delta_o);
409 + a_gl_rd += a_gl_rd_delta_o * slice_row;
410 + // Shared write index of current thread.
411 + int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
412 + (threadIdx.x % a_gl_rd_delta_o);
413 + // Shared read index.
414 + int a_sh_rd =
415 + a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
416 + a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
417 +
418 + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
419 + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
420 + b_gl_rd += b_sh_stride * slice_col;
421 + b_gl_rd += b_gl_rd_delta_o * slice_row;
422 + int b_sh_wr = threadIdx.x * b_thread_vecs;
423 + int b_sh_rd = threadIdx.x * b_thread_vecs;
424 +
425 + // For act_order
426 + constexpr int k_iter_size = tb_k / b_sh_wr_iters;
427 + int slice_k_start = tb_k * slice_row;
428 + int slice_k_finish = slice_k_start + tb_k * slice_iters;
429 + int slice_k_start_shared_fetch = slice_k_start;
430 + int slice_n_offset = act_s_col_tb_stride * slice_col;
431 +
432 + // No act_order
433 + int s_gl_rd;
434 + if constexpr (!has_act_order) {
435 + if constexpr (group_blocks == -1) {
436 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
437 + } else {
438 + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
439 + s_sh_stride * slice_col + threadIdx.x;
440 + }
441 + }
442 + int s_sh_wr = threadIdx.x;
443 + bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
444 +
445 + // We use a different scale layout for grouped and column-wise quantization as
446 + // we scale a `half2` tile in column-major layout in the former and in
447 + // row-major in the latter case.
448 + int s_sh_rd;
449 + if constexpr (group_blocks != -1)
450 + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
451 + (threadIdx.x % 32) / 4;
452 + else
453 + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
454 + (threadIdx.x % 32) % 4;
455 +
456 + int sh_first_group_id = -1;
457 + int sh_num_groups = -1;
458 + constexpr int sh_max_num_groups = 32;
459 +
460 + int shs_size;
461 + if constexpr (has_act_order)
462 + shs_size = sh_max_num_groups * s_sh_stride + threads;
463 + else
464 + shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
465 +
466 + extern __shared__ int4 sh[];
467 + // Shared memory storage for global fetch pipelines.
468 + int4* sh_a = sh;
469 + int4* sh_b = sh_a + (stages * a_sh_stage);
470 + int4* sh_g_idx = sh_b + (stages * b_sh_stage);
471 + int4* sh_s = sh_g_idx + (stages * g_idx_stage);
472 + int* sh_sorted = (int*)(sh_s + shs_size);
473 +
474 + // Precompute which thread should not read memory in which iterations; this is
475 + // needed if there are more threads than required for a certain tilesize or
476 + // when the batchsize is not a multiple of 16.
477 + bool a_sh_wr_pred[a_sh_wr_iters];
478 + #pragma unroll
479 + for (int i = 0; i < a_sh_wr_iters; i++) {
480 + int a_idx = a_sh_wr_delta * i + a_sh_wr;
481 + int row = a_idx / a_gl_rd_delta_o;
482 + if (row >= prob_m) {
483 + a_sh_wr_pred[i] = false;
484 + } else {
485 + a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
486 + }
487 + }
488 +
489 + // To ensure that writing and reading A tiles to/from shared memory, the
490 + // latter in fragment format, is fully bank conflict free, we need to use a
491 + // rather fancy XOR-based layout. The key here is that neither reads nor
492 + // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
493 + // same shared memory banks. Further, it seems (based on NSight-Compute) that
494 + // each warp must also write a consecutive memory segment?
495 + auto transform_a = [&](int i) {
496 + int row = i / a_gl_rd_delta_o;
497 + return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
498 + };
499 + // Since the computation of this remapping is non-trivial and, due to our main
500 + // loop unrolls, all shared memory accesses are static, we simply precompute
501 + // both transformed reads and writes.
502 + int a_sh_wr_trans[a_sh_wr_iters];
503 + #pragma unroll
504 + for (int i = 0; i < a_sh_wr_iters; i++)
505 + a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
506 + int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
507 + #pragma unroll
508 + for (int i = 0; i < b_sh_wr_iters; i++) {
509 + #pragma unroll
510 + for (int j = 0; j < thread_m_blocks; j++)
511 + a_sh_rd_trans[i][j] =
512 + transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
513 + }
514 +
515 + // Since B-accesses have non-constant stride they have to be computed at
516 + // runtime; we break dependencies between subsequent accesses with a tile by
517 + // maintining multiple pointers (we have enough registers), a tiny
518 + // optimization.
519 + const int4* B_ptr[b_sh_wr_iters];
520 + #pragma unroll
521 + for (int i = 0; i < b_sh_wr_iters; i++)
522 + B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
523 +
524 + // Register storage for double buffer of shared memory reads.
525 + FragA frag_a[2][thread_m_blocks];
526 + I4 frag_b_quant[2][b_thread_vecs];
527 + FragC frag_c[thread_m_blocks][4][2];
528 + FragS frag_s[2][4]; // No act-order
529 + FragS act_frag_s[2][4][4]; // For act-order
530 +
531 + // Zero accumulators.
532 + auto zero_accums = [&]() {
533 + #pragma unroll
534 + for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
535 + reinterpret_cast<float*>(frag_c)[i] = 0;
536 + };
537 +
538 + auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
539 + int last_group_id) {
540 + sh_first_group_id = first_group_id;
541 + sh_num_groups = last_group_id - first_group_id + 1;
542 +
543 + if (sh_num_groups < sh_max_num_groups) {
544 + sh_num_groups = sh_max_num_groups;
545 + }
546 +
547 + if (sh_first_group_id + sh_num_groups > num_groups) {
548 + sh_num_groups = num_groups - sh_first_group_id;
549 + }
550 +
551 + int row_offset = first_group_id * s_gl_stride;
552 +
553 + if (is_async) {
554 + for (int i = 0; i < sh_num_groups; i++) {
555 + if (threadIdx.x < s_sh_stride) {
556 + cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
557 + &scales_ptr[row_offset + (i * s_gl_stride) +
558 + slice_n_offset + threadIdx.x]);
559 + }
560 + }
561 + } else {
562 + for (int i = 0; i < sh_num_groups; i++) {
563 + if (threadIdx.x < s_sh_stride) {
564 + sh_s[(i * s_sh_stride) + threadIdx.x] =
565 + scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
566 + threadIdx.x];
567 + }
568 + }
569 + }
570 + };
571 + // Asynchronously fetch the next A, B and s tile from global to the next
572 + // shared memory pipeline location.
573 + auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
574 + if (pred) {
575 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
576 + #pragma unroll
577 + for (int i = 0; i < a_sh_wr_iters; i++) {
578 + int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off;
579 + int row = a_idx / a_gl_stride;
580 + int sorted_row =
581 + replicate_input ? sorted_ids[row] / topk : sorted_ids[row];
582 + int new_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride;
583 + if (sorted_row < tot_m * (replicate_input ? 1 : topk) &&
584 + new_idx < a_gl_stride * tot_m * (replicate_input ? 1 : topk)) {
585 + cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[new_idx],
586 + a_sh_wr_pred[i]);
587 + }
588 + }
589 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
590 + #pragma unroll
591 + for (int i = 0; i < b_sh_wr_iters; i++) {
592 + #pragma unroll
593 + for (int j = 0; j < b_thread_vecs; j++) {
594 + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
595 + }
596 + B_ptr[i] += b_gl_rd_delta_o;
597 + }
598 +
599 + if constexpr (has_act_order) {
600 + // Fetch g_idx thread-block portion
601 + int full_pipe = a_off;
602 + int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
603 + if (cur_k < prob_k && cur_k < slice_k_finish) {
604 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
605 +
606 + int4 const* cur_g_idx_stage_ptr =
607 + reinterpret_cast<int4 const*>(&g_idx[cur_k]);
608 +
609 + if (threadIdx.x < g_idx_stage) {
610 + cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
611 + &cur_g_idx_stage_ptr[threadIdx.x]);
612 + }
613 + }
614 + } else {
615 + if constexpr (group_blocks != -1) {
616 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
617 +
618 + if constexpr (group_blocks >= thread_k_blocks) {
619 + // Only fetch scales if this tile starts a new group
620 + if (pipe % (group_blocks / thread_k_blocks) == 0) {
621 + if (s_sh_wr_pred) {
622 + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
623 + }
624 + s_gl_rd += s_gl_rd_delta;
625 + }
626 + } else {
627 + for (int i = 0; i < s_tb_groups; i++) {
628 + if (s_sh_wr_pred) {
629 + cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
630 + &scales_ptr[s_gl_rd]);
631 + }
632 + s_gl_rd += s_gl_rd_delta;
633 + }
634 + }
635 + }
636 + }
637 + }
638 + // Insert a fence even when we are winding down the pipeline to ensure that
639 + // waiting is also correct at this point.
640 + cp_async_fence();
641 + };
642 +
643 + // TODO we are currently hitting illegal memory accesses when fetching
644 + // sorted_ids to shared data: fix this
645 + auto fetch_sorted_ids_to_shared = [&]() {
646 + const int mpt = ceildiv(prob_m, threads);
647 + for (int i = 0; i < mpt; i++) {
648 + if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
649 + sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
650 + sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
651 + }
652 + }
653 + };
654 +
655 + // Wait until the next thread tile has been loaded to shared memory.
656 + auto wait_for_stage = [&]() {
657 + // We only have `stages - 2` active fetches since we are double buffering
658 + // and can only issue the next fetch when it is guaranteed that the previous
659 + // shared memory load is fully complete (as it may otherwise be
660 + // overwritten).
661 + cp_async_wait<stages - 2>();
662 + __syncthreads();
663 + };
664 +
665 + // Load the next sub-tile from the current location in the shared memory pipe
666 + // into the current register buffer.
667 + auto fetch_to_registers = [&](int k, int pipe) {
668 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
669 + #pragma unroll
670 + for (int i = 0; i < thread_m_blocks; i++)
671 + ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
672 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
673 +
674 + #pragma unroll
675 + for (int i = 0; i < b_thread_vecs; i++) {
676 + frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
677 + &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
678 + }
679 + };
680 +
681 + bool is_same_group[stages];
682 + int same_group_id[stages];
683 +
684 + auto init_same_group = [&](int pipe) {
685 + if constexpr (!has_act_order) {
686 + is_same_group[pipe] = false;
687 + same_group_id[pipe] = 0;
688 + return;
689 + }
690 +
691 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
692 + int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
693 +
694 + int group_id_1 = sh_g_idx_int_ptr[0];
695 + int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
696 +
697 + is_same_group[pipe] = group_id_1 == group_id_2;
698 + same_group_id[pipe] = group_id_1;
699 + };
700 +
701 + auto fetch_scales_to_registers = [&](int k, int full_pipe) {
702 + int pipe = full_pipe % stages;
703 +
704 + if constexpr (!has_act_order) {
705 + // No act-order case
706 + if constexpr (group_blocks != -1) {
707 + if constexpr (group_blocks >= thread_k_blocks) {
708 + int4* sh_s_stage =
709 + sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
710 + (pipe / (group_blocks / thread_k_blocks)));
711 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
712 + } else {
713 + int warp_id = threadIdx.x / 32;
714 + int n_warps = thread_n_blocks / 4;
715 +
716 + int warp_row = warp_id / n_warps;
717 +
718 + int cur_k = warp_row * 16;
719 + cur_k += k_iter_size * (k % b_sh_wr_iters);
720 +
721 + int k_blocks = cur_k / 16;
722 + int cur_group_id = k_blocks / group_blocks;
723 +
724 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
725 +
726 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
727 + sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
728 + }
729 + }
730 +
731 + return;
732 + }
733 +
734 + // Act-order case
735 +
736 + // Determine K of the "current" thread-block
737 + int cur_k = slice_k_start + tb_k * full_pipe;
738 + if (cur_k >= prob_k || cur_k >= slice_k_finish) {
739 + return;
740 + }
741 +
742 + // Reset (to current thread-block) since we read g_idx portion from the
743 + // shared memory
744 + cur_k = 0;
745 +
746 + // Progress to current iteration
747 + cur_k += k_iter_size * (k % b_sh_wr_iters);
748 +
749 + // Determine "position" inside the thread-block (based on warp and
750 + // thread-id)
751 + int warp_id = threadIdx.x / 32;
752 + int n_warps =
753 + thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
754 +
755 + int warp_row = warp_id / n_warps;
756 + int warp_col = warp_id % n_warps;
757 +
758 + cur_k += warp_row * 16;
759 +
760 + int th_id = threadIdx.x % 32;
761 + cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
762 +
763 + int s_col_shift =
764 + /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
765 + (th_id / 4) * act_s_col_stride;
766 +
767 + if (is_same_group[pipe]) {
768 + if (k % 2 == 0) {
769 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
770 + sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
771 + s_col_shift];
772 + } else {
773 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
774 + *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
775 + }
776 +
777 + for (int i = 1; i < 4; i++) {
778 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
779 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
780 + }
781 + return;
782 + }
783 +
784 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
785 + int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
786 +
787 + constexpr int k_frag_offsets[4] = {0, 1, 8,
788 + 9}; // Tensor core offsets per thread
789 +
790 + #pragma unroll
791 + for (int i = 0; i < 4; i++) {
792 + int actual_k = cur_k + k_frag_offsets[i];
793 +
794 + int group_id = sh_g_idx_int_ptr[actual_k];
795 + int rel_group_id = group_id - sh_first_group_id;
796 +
797 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
798 + sh_s[rel_group_id * s_sh_stride + s_col_shift];
799 + }
800 + };
801 +
802 + // Execute the actual tensor core matmul of a sub-tile.
803 + auto matmul = [&](int k) {
804 + // We have the m dimension as the inner loop in order to encourage overlapping
805 + // dequantization and matmul operations.
806 + #pragma unroll
807 + for (int j = 0; j < 4; j++) {
808 + int b_quant_0, b_quant_1;
809 + if constexpr (w_type.size_bits() == 4) {
810 + b_quant_0 = frag_b_quant[k % 2][0][j];
811 + b_quant_1 = b_quant_0 >> 8;
812 + } else {
813 + static_assert(w_type.size_bits() == 8);
814 + int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
815 + b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
816 + b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
817 + }
818 +
819 + FragB frag_b0 = dequant<w_type_id>(b_quant_0);
820 + FragB frag_b1 = dequant<w_type_id>(b_quant_1);
821 +
822 + // Apply scale to frag_b0
823 + if constexpr (has_act_order) {
824 + scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
825 + act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
826 + } else {
827 + if constexpr (group_blocks != -1) {
828 + scale(frag_b0, frag_s[k % 2][j], 0);
829 + }
830 + }
831 +
832 + // Apply scale to frag_b1
833 + if constexpr (has_act_order) {
834 + scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
835 + act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
836 +
837 + } else {
838 + if constexpr (group_blocks != -1) {
839 + scale(frag_b1, frag_s[k % 2][j], 1);
840 + }
841 + }
842 +
843 + #pragma unroll
844 + for (int i = 0; i < thread_m_blocks; i++) {
845 + mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
846 + mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
847 + }
848 + }
849 + };
850 +
851 + // Since we slice across the k dimension of a tile in order to increase the
852 + // number of warps while keeping the n dimension of a tile reasonable, we have
853 + // multiple warps that accumulate their partial sums of the same output
854 + // location; which we have to reduce over in the end. We do in shared memory.
855 + auto thread_block_reduce = [&]() {
856 + constexpr int red_off = threads / b_sh_stride_threads / 2;
857 + if (red_off >= 1) {
858 + int red_idx = threadIdx.x / b_sh_stride_threads;
859 + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
860 + constexpr int red_sh_delta = b_sh_stride_threads;
861 + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
862 + (threadIdx.x % b_sh_stride_threads);
863 +
864 + // Parallel logarithmic shared memory reduction. We make sure to avoid any
865 + // unnecessary read or write iterations, e.g., for two warps we write only
866 + // once by warp 1 and read only once by warp 0.
867 +
868 + #pragma unroll
869 + for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
870 + #pragma unroll
871 + for (int i = red_off; i > 0; i /= 2) {
872 + if (i <= red_idx && red_idx < 2 * i) {
873 + #pragma unroll
874 + for (int j = 0; j < 4 * 2; j++) {
875 + int red_sh_wr =
876 + red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
877 + if (i < red_off) {
878 + float* c_rd =
879 + reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
880 + float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
881 + #pragma unroll
882 + for (int k = 0; k < 4; k++)
883 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
884 + c_rd[k] + c_wr[k];
885 + }
886 + sh[red_sh_wr] =
887 + reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
888 + }
889 + }
890 + __syncthreads();
891 + }
892 + if (red_idx == 0) {
893 + #pragma unroll
894 + for (int i = 0; i < 4 * 2; i++) {
895 + float* c_rd =
896 + reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
897 + #pragma unroll
898 + for (int j = 0; j < 4; j++)
899 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
900 + c_rd[j];
901 + }
902 + }
903 + __syncthreads();
904 + }
905 + }
906 + };
907 +
908 + // Since multiple threadblocks may process parts of the same column slice, we
909 + // finally have to globally reduce over the results. As the striped
910 + // partitioning minimizes the number of such reductions and our outputs are
911 + // usually rather small, we perform this reduction serially in L2 cache.
912 + auto global_reduce = [&](bool first = false, bool last = false) {
913 + // We are very careful here to reduce directly in the output buffer to
914 + // maximize L2 cache utilization in this step. To do this, we write out
915 + // results in FP16 (but still reduce with FP32 compute).
916 + constexpr int active_threads = 32 * thread_n_blocks / 4;
917 + if (threadIdx.x < active_threads) {
918 + int c_gl_stride = prob_n / 8;
919 + int c_gl_wr_delta_o = 8 * c_gl_stride;
920 + int c_gl_wr_delta_i = 4 * (active_threads / 32);
921 + int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
922 + 4 * (threadIdx.x / 32) + threadIdx.x % 4;
923 + c_gl_wr += (2 * thread_n_blocks) * slice_col;
924 + constexpr int c_sh_wr_delta = active_threads;
925 + int c_sh_wr = threadIdx.x;
926 +
927 + int row = (threadIdx.x % 32) / 4;
928 +
929 + if (!first) {
930 + // Interestingly, doing direct global accesses here really seems to mess up
931 + // the compiler and lead to slowdowns, hence we also use async-copies even
932 + // though these fetches are not actually asynchronous.
933 + #pragma unroll
934 + for (int i = 0; i < thread_m_blocks * 4; i++) {
935 + int c_idx =
936 + c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
937 + int sorted_row = sorted_ids[c_idx / c_gl_stride];
938 + int new_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride;
939 + cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i], &C[new_idx],
940 + sorted_row < tot_m * topk &&
941 + (8 * (i / 2) + row < prob_m &&
942 + (i < (thread_m_blocks - 1) * 4 ||
943 + sorted_ids[8 * (i / 2) + row] < tot_m * topk)));
944 + }
945 + cp_async_fence();
946 + cp_async_wait<0>();
947 + }
948 +
949 + #pragma unroll
950 + for (int i = 0; i < thread_m_blocks * 4; i++) {
951 + if (8 * (i / 2) + row < prob_m &&
952 + (i < (thread_m_blocks - 1) * 4 ||
953 + sorted_ids[8 * (i / 2) + row] < tot_m * topk)) {
954 + if (!first) {
955 + int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
956 + #pragma unroll
957 + for (int j = 0; j < 2 * 4; j++) {
958 + reinterpret_cast<float*>(
959 + &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
960 + __half2float(reinterpret_cast<__half*>(&c_red)[j]);
961 + }
962 + }
963 + if (!last) {
964 + int4 c;
965 + #pragma unroll
966 + for (int j = 0; j < 2 * 4; j++) {
967 + reinterpret_cast<__half*>(&c)[j] =
968 + __float2half(reinterpret_cast<float*>(
969 + &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
970 + }
971 + int c_idx =
972 + c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
973 + int row = sorted_ids[c_idx / c_gl_stride];
974 + if (row < tot_m * topk) {
975 + int new_idx = row * c_gl_stride + c_idx % c_gl_stride;
976 + C[new_idx] = c;
977 + }
978 + }
979 + }
980 + }
981 + }
982 + };
983 +
984 + // Write out the reduce final result in the correct layout. We only actually
985 + // reshuffle matrix fragments in this step, the reduction above is performed
986 + // in fragment layout.
987 + auto write_result = [&]() {
988 + int c_gl_stride = prob_n / 8;
989 + constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
990 + int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
991 + constexpr int c_sh_rd_delta =
992 + c_sh_stride * (threads / (2 * thread_n_blocks));
993 +
994 + int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
995 + (threadIdx.x % (2 * thread_n_blocks));
996 + c_gl_wr += (2 * thread_n_blocks) * slice_col;
997 + int c_sh_wr =
998 + (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
999 + c_sh_wr += 32 * (threadIdx.x / 32);
1000 + int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1001 + (threadIdx.x % (2 * thread_n_blocks));
1002 +
1003 + int c_gl_wr_end = c_gl_stride * prob_m;
1004 +
1005 + // We first reorder in shared memory to guarantee the most efficient final
1006 + // global write patterns
1007 + auto write = [&](int idx, float c0, float c1, FragS& s) {
1008 + half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1009 +
1010 + // For per-column quantization we finally apply the scale here (only for
1011 + // 4-bit)
1012 + if constexpr (!has_act_order && group_blocks == -1 &&
1013 + w_type.size_bits() == 4) {
1014 + res = __hmul2(res, s[0]);
1015 + }
1016 +
1017 + ((half2*)sh)[idx] = res;
1018 + };
1019 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1020 + #pragma unroll
1021 + for (int i = 0; i < thread_m_blocks; i++) {
1022 + #pragma unroll
1023 + for (int j = 0; j < 4; j++) {
1024 + int wr = c_sh_wr + 8 * j;
1025 + write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
1026 + frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
1027 + write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
1028 + frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
1029 + write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
1030 + frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
1031 + write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
1032 + frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
1033 + }
1034 + c_sh_wr += 16 * (4 * c_sh_stride);
1035 + }
1036 + }
1037 + __syncthreads();
1038 +
1039 + #pragma unroll
1040 + for (int i = 0;
1041 + i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1042 + i++) {
1043 + if (c_gl_wr < c_gl_wr_end) {
1044 + int row = sorted_ids[c_gl_wr / c_gl_stride];
1045 + if (row < tot_m * topk) {
1046 + int off = row * c_gl_stride + c_gl_wr % c_gl_stride;
1047 + if (!apply_weights) {
1048 + C[off] = sh[c_sh_rd];
1049 + } else {
1050 + __half* ctrg = reinterpret_cast<__half*>(&C[off]);
1051 + __half* csrc = reinterpret_cast<__half*>(&sh[c_sh_rd]);
1052 + for (int j = 0; j < 8; ++j) {
1053 + ctrg[j] = __float2half(topk_weights[row] * __half2float(csrc[j]));
1054 + }
1055 + }
1056 + c_gl_wr += c_gl_wr_delta;
1057 + c_sh_rd += c_sh_rd_delta;
1058 + }
1059 + }
1060 + }
1061 + };
1062 +
1063 + // Start global fetch and register load pipelines.
1064 + auto start_pipes = [&]() {
1065 + // TODO re-enable after fixing this function
1066 + // fetch_sorted_ids_to_shared();
1067 + // __syncthreads();
1068 +
1069 + #pragma unroll
1070 + for (int i = 0; i < stages - 1; i++) {
1071 + if (has_act_order && i == 0) {
1072 + int last_g_idx = slice_k_start + stages * tb_k * 2;
1073 + if (last_g_idx >= prob_k) {
1074 + last_g_idx = prob_k - 1;
1075 + }
1076 + fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1077 + }
1078 + fetch_to_shared(i, i, i < slice_iters);
1079 + }
1080 +
1081 + zero_accums();
1082 + wait_for_stage();
1083 + init_same_group(0);
1084 + fetch_to_registers(0, 0);
1085 + fetch_scales_to_registers(0, 0);
1086 + a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1087 + slice_k_start_shared_fetch += tb_k * (stages - 1);
1088 + };
1089 + if (slice_iters) {
1090 + start_pipes();
1091 + }
1092 +
1093 + // Main loop.
1094 + while (slice_iters) {
1095 + // We unroll over both the global fetch and the register load pipeline to
1096 + // ensure all shared memory accesses are static. Note that both pipelines
1097 + // have even length meaning that the next iteration will always start at
1098 + // index 0.
1099 + #pragma unroll
1100 + for (int pipe = 0; pipe < stages;) {
1101 + #pragma unroll
1102 + for (int k = 0; k < b_sh_wr_iters; k++) {
1103 + fetch_to_registers(k + 1, pipe % stages);
1104 + fetch_scales_to_registers(k + 1, pipe);
1105 + if (k == b_sh_wr_iters - 2) {
1106 + fetch_to_shared((pipe + stages - 1) % stages, pipe,
1107 + slice_iters >= stages);
1108 + pipe++;
1109 + wait_for_stage();
1110 + init_same_group(pipe % stages);
1111 + }
1112 + matmul(k);
1113 + }
1114 + slice_iters--;
1115 + if (slice_iters == 0) {
1116 + break;
1117 + }
1118 + }
1119 +
1120 + a_gl_rd += a_gl_rd_delta_o * stages;
1121 + slice_k_start += tb_k * stages;
1122 + slice_k_start_shared_fetch += tb_k * stages;
1123 +
1124 + if constexpr (has_act_order) {
1125 + int first_group_id = g_idx[slice_k_start];
1126 + int last_g_idx = slice_k_start + stages * tb_k * 2;
1127 + if (last_g_idx >= prob_k) {
1128 + last_g_idx = prob_k - 1;
1129 + }
1130 + int last_group_id = g_idx[last_g_idx];
1131 + if (last_group_id >= sh_first_group_id + sh_num_groups) {
1132 + fetch_scales_to_shared(false, first_group_id, last_group_id);
1133 + __syncthreads();
1134 + }
1135 + }
1136 +
1137 + // Process results and, if necessary, proceed to the next column slice.
1138 + // While this pattern may not be the most readable, other ways of writing
1139 + // the loop seemed to noticeably worse performance after compilation.
1140 + if (slice_iters == 0) {
1141 + cp_async_wait<0>();
1142 + bool last = slice_idx == slice_count - 1;
1143 + if constexpr (!has_act_order && group_blocks == -1) {
1144 + if constexpr (w_type.size_bits() == 8) {
1145 + if (s_sh_wr_pred) {
1146 + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1147 + }
1148 + cp_async_fence();
1149 + } else {
1150 + // For 4-bit per-column scales, we only fetch them here in the
1151 + // final step before write-out
1152 + if (last) {
1153 + if (s_sh_wr_pred) {
1154 + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1155 + }
1156 + cp_async_fence();
1157 + }
1158 + }
1159 + }
1160 +
1161 + thread_block_reduce();
1162 + if constexpr (!has_act_order && group_blocks == -1) {
1163 + if constexpr (w_type.size_bits() == 8) {
1164 + cp_async_wait<0>();
1165 + __syncthreads();
1166 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1167 + reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1168 + reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1169 + }
1170 +
1171 + } else {
1172 + if (last) {
1173 + cp_async_wait<0>();
1174 + __syncthreads();
1175 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1176 + reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1177 + reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1178 + }
1179 + }
1180 + }
1181 + }
1182 +
1183 + // For 8-bit channelwise, we apply the scale before the global reduction
1184 + // that converts the fp32 results to fp16 (so that we avoid possible
1185 + // overflow in fp16)
1186 + if constexpr (!has_act_order && group_blocks == -1 &&
1187 + w_type.size_bits() == 8) {
1188 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1189 + #pragma unroll
1190 + for (int i = 0; i < thread_m_blocks; i++) {
1191 + #pragma unroll
1192 + for (int j = 0; j < 4; j++) {
1193 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1194 + frag_s[j / 2][2 * (j % 2) + 0]);
1195 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1196 + frag_s[j / 2][2 * (j % 2) + 0]);
1197 +
1198 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1199 + frag_s[j / 2][2 * (j % 2) + 1]);
1200 + scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1201 + frag_s[j / 2][2 * (j % 2) + 1]);
1202 + }
1203 + }
1204 + }
1205 + }
1206 +
1207 + if (slice_count > 1) { // only globally reduce if there is more than one
1208 + // block in a slice
1209 + barrier_acquire(&locks[slice_col], slice_idx);
1210 + global_reduce(slice_idx == 0, last);
1211 + barrier_release(&locks[slice_col], last);
1212 + }
1213 + if (last) // only the last block in a slice actually writes the result
1214 + write_result();
1215 + slice_row = 0;
1216 + slice_col_par++;
1217 + slice_col++;
1218 + init_slice();
1219 + if (slice_iters) {
1220 + a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1221 + (threadIdx.x % a_gl_rd_delta_o);
1222 + #pragma unroll
1223 + for (int i = 0; i < b_sh_wr_iters; i++)
1224 + B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1225 + if (slice_col == 0) {
1226 + #pragma unroll
1227 + for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1228 + }
1229 +
1230 + // Update slice k/n for scales loading
1231 + if constexpr (has_act_order) {
1232 + slice_k_start = tb_k * slice_row;
1233 + slice_k_finish = slice_k_start + tb_k * slice_iters;
1234 + slice_k_start_shared_fetch = slice_k_start;
1235 + slice_n_offset = act_s_col_tb_stride * slice_col;
1236 +
1237 + } else {
1238 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1239 + }
1240 + start_pipes();
1241 + }
1242 + }
1243 + }
1244 +}
1245 +
1246 +template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1247 + const int threads, // number of threads in a threadblock
1248 + const int thread_n_blocks, // same for n dimension (output)
1249 + const int thread_k_blocks, // same for k dimension (reduction)
1250 + const int stages, // number of stages for the async global->shared
1251 + // fetch pipeline
1252 + const bool has_act_order, // whether act_order is enabled
1253 + const int group_blocks = -1 // number of consecutive 16x16 blocks
1254 + // with a separate quantization scale
1255 + >
1256 +__global__ void MarlinMoE(
1257 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
1258 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1259 + int4* __restrict__ C, // fp16 output buffer of shape mxn
1260 + const int* __restrict__ sorted_ids_base, // int32 sorted ids of experts
1261 + const float* __restrict__ topk_weights, // float topk weights
1262 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1263 + // (k/groupsize)xn
1264 + const int* __restrict__ g_idx, // int32 group indices of shape k
1265 + const int* __restrict__ expert_offsets,
1266 + int num_groups, // number of scale groups per output channel
1267 + int expert_idx, // idx of current expert
1268 + int num_experts, // number of experts
1269 + int topk, // topk parameter of moe
1270 + int prob_m, // batch dimension m
1271 + int prob_n, // output dimension n
1272 + int prob_k, // reduction dimension k
1273 + int tot_m, // total number of rows in A and C
1274 + int* locks, // extra global storage for barrier synchronization
1275 + bool replicate_input, // do we use the same input for each expert?
1276 + bool apply_weights, // apply weights to output
1277 + int current_m_block, // current m block to start kernel computation from
1278 + int max_par, // maximum parallelism
1279 + int cfg_max_m_blocks // upper bound on m blocks
1280 +) {
1281 + int m_block_ctr = current_m_block;
1282 +
1283 + const int* sorted_ids_expert =
1284 + sorted_ids_base + expert_offsets[expert_idx] + m_block_ctr * 4 * max_par;
1285 + int tot_its = expert_offsets[expert_idx + 1] - expert_offsets[expert_idx];
1286 + if (tot_its == 0) {
1287 + return;
1288 + }
1289 + int tot_m_blocks = ceildiv(tot_its, 16);
1290 + int pad = 16 * tot_m_blocks - tot_its;
1291 +
1292 + if (m_block_ctr >= tot_m_blocks) {
1293 + return;
1294 + }
1295 +
1296 + int max_block = tot_m_blocks - m_block_ctr;
1297 + prob_m = tot_its - 16 * m_block_ctr;
1298 +
1299 + int par = 1;
1300 + if (max_block > cfg_max_m_blocks) {
1301 + // Note that parallel > 1 currently only works for inputs without any
1302 + // padding
1303 + par = (16 * max_block - pad) / (16 * cfg_max_m_blocks);
1304 + if (par > max_par) par = max_par;
1305 + prob_m = (16 * cfg_max_m_blocks) * par;
1306 + m_block_ctr += cfg_max_m_blocks * (par - 1);
1307 + max_block = cfg_max_m_blocks;
1308 + }
1309 +
1310 + if (max_block == 1) {
1311 + MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1312 + stages, has_act_order, group_blocks>(
1313 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1314 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1315 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1316 + current_m_block);
1317 + } else if (max_block == 2) {
1318 + MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1319 + stages, has_act_order, group_blocks>(
1320 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1321 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1322 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1323 + current_m_block);
1324 + } else if (max_block == 3) {
1325 + MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1326 + stages, has_act_order, group_blocks>(
1327 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1328 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1329 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1330 + current_m_block);
1331 + } else {
1332 + MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1333 + stages, has_act_order, group_blocks>(
1334 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1335 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1336 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1337 + current_m_block);
1338 + }
1339 +}
1340 +
1341 +#else
1342 +
1343 +template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1344 + const int threads, // number of threads in a threadblock
1345 + const int thread_n_blocks, // same for n dimension (output)
1346 + const int thread_k_blocks, // same for k dimension (reduction)
1347 + const int stages, // number of stages for the async global->shared
1348 + // fetch pipeline
1349 + const bool has_act_order, // whether act_order is enabled
1350 + const int group_blocks = -1 // number of consecutive 16x16 blocks
1351 + // with a separate quantization scale
1352 + >
1353 +__global__ void MarlinMoE(
1354 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
1355 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1356 + int4* __restrict__ C, // fp16 output buffer of shape mxn
1357 + const int* __restrict__ sorted_ids, // int32 sorted ids of experts
1358 + const float* __restrict__ topk_weights, // float topk weights
1359 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1360 + // (k/groupsize)xn
1361 + const int* __restrict__ g_idx, // int32 group indices of shape k
1362 + const int* __restrict__ expert_offsets,
1363 + int num_groups, // number of scale groups per output channel
1364 + int expert_idx, // idx of current expert
1365 + int num_experts, // number of experts
1366 + int topk, // topk parameter of moe
1367 + int prob_m, // batch dimension m
1368 + int prob_n, // output dimension n
1369 + int prob_k, // reduction dimension k
1370 + int tot_m, // total number of rows in A and C
1371 + int* locks, // extra global storage for barrier synchronization
1372 + bool replicate_input, // do we use the same input for each expert?
1373 + bool apply_weights, // apply weights to output
1374 + int current_m_block, // current m block to start kernel computation from
1375 + int max_par, // maximum parallelism
1376 + int cfg_max_m_blocks // upper bound on m blocks
1377 +
1378 +) {
1379 + // Marlin is not implemented yet for SM < 8.0
1380 + assert(false);
1381 + return;
1382 +}
1383 +
1384 +#endif
1385 +
1386 +// 8 warps are a good choice since every SM has 4 schedulers and having more
1387 +// than 1 warp per schedule allows some more latency hiding. At the same time,
1388 +// we want relatively few warps to have many registers per warp and small tiles.
1389 +const int USER_THREADS =
1390 + 256; // Note: This is only used with user-provided thread_k/n
1391 +const int STAGES = 4; // 4 pipeline stages fit into shared memory
1392 +// const int SHARED_MEM =
1393 +// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1394 +
1395 +static constexpr int min_thread_n = 64;
1396 +static constexpr int min_thread_k = 64;
1397 +
1398 +#define __CALL_IF_MOE(W_TYPE, THREAD_N_BLOCKS, THREAD_K_BLOCKS, HAS_ACT_ORDER, \
1399 + GROUP_BLOCKS, NUM_THREADS) \
1400 + else if (q_type == W_TYPE && thread_n_blocks == THREAD_N_BLOCKS && \
1401 + thread_k_blocks == THREAD_K_BLOCKS && \
1402 + has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1403 + num_threads == NUM_THREADS) { \
1404 + cudaFuncSetAttribute( \
1405 + MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1406 + STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1407 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1408 + MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1409 + STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1410 + <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1411 + A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1412 + g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1413 + num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1414 + replicate_input, apply_weights, m_block, max_par, \
1415 + cfg_max_m_blocks); \
1416 + }
1417 +
1418 +#define GPTQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1419 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1420 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1421 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1422 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1423 + __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1424 +
1425 +} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu new
+29
@@ -0,0 +1,29 @@
1 +#include "marlin_moe_kernel_ku4b8.h"
2 +
3 +namespace marlin_moe {
4 +
5 +// We return bool so we can create these different kernel calls as a sequence
6 +// of if-elseif's.
7 +bool call_marlin_moe_kernel_ku4b8(
8 + vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9 + bool has_act_order, int group_blocks, int num_threads, int blocks,
10 + int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11 + const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12 + const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13 + int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14 + int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15 + bool replicate_input, bool apply_weights, int m_block, int max_par,
16 + int cfg_max_m_blocks) {
17 + if (false) {
18 + }
19 + GPTQ_CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
20 + GPTQ_CALL_IF_MOE(vllm::kU4B8, 8, 8, 256)
21 + GPTQ_CALL_IF_MOE(vllm::kU4B8, 8, 4, 128)
22 + GPTQ_CALL_IF_MOE(vllm::kU4B8, 4, 8, 128)
23 + else {
24 + return false;
25 + }
26 + return true;
27 +}
28 +
29 +} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h new
+20
@@ -0,0 +1,20 @@
1 +#pragma once
2 +
3 +#include "marlin_moe_kernel.h"
4 +
5 +namespace marlin_moe {
6 +
7 +// We return bool so we can create these different kernel calls as a sequence
8 +// of if-elseif's.
9 +bool call_marlin_moe_kernel_ku4b8(
10 + vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
11 + bool has_act_order, int group_blocks, int num_threads, int blocks,
12 + int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
13 + const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
14 + const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
15 + int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
16 + int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
17 + bool replicate_input, bool apply_weights, int m_block, int max_par,
18 + int cfg_max_m_blocks);
19 +
20 +} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu new
+29
@@ -0,0 +1,29 @@
1 +#include "marlin_moe_kernel_ku8b128.h"
2 +
3 +namespace marlin_moe {
4 +
5 +// We return bool so we can create these different kernel calls as a sequence
6 +// of if-elseif's.
7 +bool call_marlin_moe_kernel_ku8b128(
8 + vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9 + bool has_act_order, int group_blocks, int num_threads, int blocks,
10 + int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11 + const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12 + const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13 + int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14 + int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15 + bool replicate_input, bool apply_weights, int m_block, int max_par,
16 + int cfg_max_m_blocks) {
17 + if (false) {
18 + }
19 + GPTQ_CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
20 + GPTQ_CALL_IF_MOE(vllm::kU8B128, 8, 8, 256)
21 + GPTQ_CALL_IF_MOE(vllm::kU8B128, 8, 4, 128)
22 + GPTQ_CALL_IF_MOE(vllm::kU8B128, 4, 8, 128)
23 + else {
24 + return false;
25 + }
26 + return true;
27 +}
28 +
29 +} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h new
+18
@@ -0,0 +1,18 @@
1 +#pragma once
2 +
3 +#include "marlin_moe_kernel.h"
4 +
5 +namespace marlin_moe {
6 +
7 +bool call_marlin_moe_kernel_ku8b128(
8 + vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9 + bool has_act_order, int group_blocks, int num_threads, int blocks,
10 + int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11 + const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12 + const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13 + int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14 + int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15 + bool replicate_input, bool apply_weights, int m_block, int max_par,
16 + int cfg_max_m_blocks);
17 +
18 +}
csrc/moe/marlin_moe_ops.cu
+26 -1427
@@ -26,6 +26,8 @@
26 #include <iostream>
27
28 #include "core/scalar_type.hpp"
29 +#include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
30 +#include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
31
32 template <typename T>
33 inline std::string str(T x) {
@@ -34,230 +36,8 @@ inline std::string str(T x) {
36
37 namespace marlin_moe {
38
37 -constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
38 -
39 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
40
41 -// Instances of `Vec` are used to organize groups of >>registers<<, as needed
42 -// for instance as inputs to tensor core operations. Consequently, all
43 -// corresponding index accesses must be compile-time constants, which is why we
44 -// extensively use `#pragma unroll` throughout the kernel code to guarantee
45 -// this.
46 -template <typename T, int n>
47 -struct Vec {
48 - T elems[n];
49 - __device__ T& operator[](int i) { return elems[i]; }
50 -};
51 -
52 -using I4 = Vec<int, 4>;
53 -
54 -// Matrix fragments for tensor core instructions; their precise layout is
55 -// documented here:
56 -// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
57 -using FragA = Vec<half2, 4>;
58 -using FragB = Vec<half2, 2>;
59 -using FragC = Vec<float, 4>;
60 -using FragS = Vec<half2, 1>; // quantization scales
61 -
62 -// Predicated asynchronous global->shared copy; used for inputs A where we apply
63 -// predication to handle batchsizes that are not multiples of 16.
64 -__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
65 - bool pred = true) {
66 - const int BYTES = 16;
67 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
68 - asm volatile(
69 - "{\n"
70 - " .reg .pred p;\n"
71 - " setp.ne.b32 p, %0, 0;\n"
72 - " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
73 - "}\n" ::"r"((int)pred),
74 - "r"(smem), "l"(glob_ptr), "n"(BYTES));
75 -}
76 -
77 -// Asynchronous global->shared copy
78 -__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
79 - const int BYTES = 16;
80 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
81 - asm volatile(
82 - "{\n"
83 - " cp.async.cg.shared.global [%0], [%1], %2;\n"
84 - "}\n" ::"r"(smem),
85 - "l"(glob_ptr), "n"(BYTES));
86 -}
87 -
88 -// Async copy fence.
89 -__device__ inline void cp_async_fence() {
90 - asm volatile("cp.async.commit_group;\n" ::);
91 -}
92 -
93 -// Wait until at most `n` async copy stages are still pending.
94 -template <int n>
95 -__device__ inline void cp_async_wait() {
96 - asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
97 -}
98 -
99 -// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
100 -// output/accumulation.
101 -__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
102 - FragC& frag_c) {
103 - const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
104 - const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
105 - float* c = reinterpret_cast<float*>(&frag_c);
106 - asm volatile(
107 - "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
108 - "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
109 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
110 - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
111 - "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
112 -}
113 -
114 -// Instruction for loading a full 16x16 matrix fragment of operand A from shared
115 -// memory, directly in tensor core layout.
116 -__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
117 - uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
118 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
119 - asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
120 - : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
121 - : "r"(smem));
122 -}
123 -
124 -// Lookup-table based 3-input logical operation; explicitly used for
125 -// dequantization as the compiler does not seem to automatically recognize it in
126 -// all cases.
127 -template <int lut>
128 -__device__ inline int lop3(int a, int b, int c) {
129 - int res;
130 - asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
131 - : "=r"(res)
132 - : "r"(a), "r"(b), "r"(c), "n"(lut));
133 - return res;
134 -}
135 -
136 -// Constructs destination register by taking bytes from 2 sources (based on
137 -// mask)
138 -template <int start_byte, int mask>
139 -__device__ inline uint32_t prmt(uint32_t a) {
140 - uint32_t res;
141 - asm volatile("prmt.b32 %0, %1, %2, %3;\n"
142 - : "=r"(res)
143 - : "r"(a), "n"(start_byte), "n"(mask));
144 - return res;
145 -}
146 -
147 -template <vllm::ScalarTypeId w_type_id>
148 -__device__ inline FragB dequant(int q);
149 -
150 -// Efficiently dequantize 4bit values packed in an int32 value into a full
151 -// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
152 -// with some small changes:
153 -// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
154 -template <>
155 -__device__ inline FragB dequant<vllm::kU4B8.id()>(int q) {
156 - const int LO = 0x000f000f;
157 - const int HI = 0x00f000f0;
158 - const int EX = 0x64006400;
159 - // Guarantee that the `(a & b) | c` operations are LOP3s.
160 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
161 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
162 - // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
163 - // directly into `SUB` and `ADD`.
164 - const int SUB = 0x64086408;
165 - const int MUL = 0x2c002c00;
166 - const int ADD = 0xd480d480;
167 - FragB frag_b;
168 - frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
169 - *reinterpret_cast<const half2*>(&SUB));
170 - frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
171 - *reinterpret_cast<const half2*>(&MUL),
172 - *reinterpret_cast<const half2*>(&ADD));
173 - return frag_b;
174 -}
175 -
176 -// Fast Int8ToFp16: Efficiently dequantize 8bit int values to fp16
177 -// Reference:
178 -// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
179 -template <>
180 -__device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
181 - static constexpr uint32_t mask_for_elt_01 = 0x5250;
182 - static constexpr uint32_t mask_for_elt_23 = 0x5351;
183 - static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
184 -
185 - uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
186 - uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
187 -
188 - static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
189 -
190 - FragB frag_b;
191 - frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
192 - *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
193 - frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
194 - *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
195 - return frag_b;
196 -}
197 -
198 -// Multiply dequantized values by the corresponding quantization scale; used
199 -// only for grouped quantization.
200 -__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
201 - half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
202 - frag_b[0] = __hmul2(frag_b[0], s);
203 - frag_b[1] = __hmul2(frag_b[1], s);
204 -}
205 -
206 -// Given 2 floats multiply by 2 scales (halves)
207 -__device__ inline void scale_float(float* c, FragS& s) {
208 - __half* s_ptr = reinterpret_cast<__half*>(&s);
209 - c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
210 - c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
211 -}
212 -
213 -// Same as above, but for act_order (each K is multiplied individually)
214 -__device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
215 - FragS& frag_s_3, FragS& frag_s_4, int i) {
216 - __half2 s_val_1_2;
217 - s_val_1_2.x = reinterpret_cast<__half*>(&frag_s_1)[i];
218 - s_val_1_2.y = reinterpret_cast<__half*>(&frag_s_2)[i];
219 -
220 - __half2 s_val_3_4;
221 - s_val_3_4.x = reinterpret_cast<__half*>(&frag_s_3)[i];
222 - s_val_3_4.y = reinterpret_cast<__half*>(&frag_s_4)[i];
223 -
224 - frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
225 - frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
226 -}
227 -
228 -// Wait until barrier reaches `count`, then lock for current threadblock.
229 -__device__ inline void barrier_acquire(int* lock, int count) {
230 - if (threadIdx.x == 0) {
231 - int state = -1;
232 - do
233 - // Guarantee that subsequent writes by this threadblock will be visible
234 - // globally.
235 - asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
236 - : "=r"(state)
237 - : "l"(lock));
238 - while (state != count);
239 - }
240 - __syncthreads();
241 -}
242 -
243 -// Release barrier and increment visitation count.
244 -__device__ inline void barrier_release(int* lock, bool reset = false) {
245 - __syncthreads();
246 - if (threadIdx.x == 0) {
247 - if (reset) {
248 - lock[0] = 0;
249 - return;
250 - }
251 - int val = 1;
252 - // Make sure that all writes since acquiring this barrier are visible
253 - // globally, while releasing the barrier.
254 - asm volatile("fence.acq_rel.gpu;\n");
255 - asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
256 - :
257 - : "l"(lock), "r"(val));
258 - }
259 -}
260 -
41 // For a given "a" of size [M,K] performs a permutation of the K columns based
42 // on the given "perm" indices.
43 __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
@@ -335,1106 +115,6 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
115 __syncthreads();
116 }
117
338 -template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
339 - const int threads, // number of threads in a threadblock
340 - const int thread_m_blocks, // number of 16x16 blocks in the m
341 - // dimension (batchsize) of the
342 - // threadblock
343 - const int thread_n_blocks, // same for n dimension (output)
344 - const int thread_k_blocks, // same for k dimension (reduction)
345 - const int stages, // number of stages for the async global->shared
346 - // fetch pipeline
347 - const bool has_act_order, // whether act_order is enabled
348 - const int group_blocks = -1 // number of consecutive 16x16 blocks
349 - // with a separate quantization scale
350 - >
351 -__device__ inline void MarlinMoESingle(
352 - const int4* __restrict__ A, // fp16 input matrix of shape mxk
353 - const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
354 - int4* __restrict__ C, // fp16 output buffer of shape mxn
355 - const int* __restrict__ sorted_ids, // int32 sorted ids of experts
356 - const float* __restrict__ topk_weights, // float topk weights
357 - const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
358 - // (k/groupsize)xn
359 - const int* __restrict__ g_idx, // int32 group indices of shape k
360 - const int* __restrict__ expert_offsets,
361 - int num_groups, // number of scale groups per output channel
362 - int expert_idx, // idx of current expert
363 - int num_experts, // number of experts
364 - int topk, // topk parameter of moe
365 - int prob_m, // batch dimension m
366 - int prob_n, // output dimension n
367 - int prob_k, // reduction dimension k
368 - int tot_m, // total number of rows in A and C
369 - int* locks, // extra global storage for barrier synchronization
370 - bool replicate_input, // do we use the same input for each expert?
371 - bool apply_weights, // apply weights to output
372 - int current_m_block // current m block to start kernel computation from
373 -) {
374 - static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
375 - constexpr int pack_factor = 32 / w_type.size_bits();
376 -
377 - // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
378 - // better partitioning with less reductions
379 - int parallel = 1;
380 - if (prob_m > 16 * thread_m_blocks) {
381 - parallel = prob_m / (16 * thread_m_blocks);
382 - prob_m = 16 * thread_m_blocks;
383 - }
384 -
385 - int k_tiles = prob_k / 16 / thread_k_blocks;
386 - int n_tiles = prob_n / 16 / thread_n_blocks;
387 - int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
388 -
389 - if constexpr (!has_act_order && group_blocks != -1) {
390 - if (group_blocks >= thread_k_blocks) {
391 - // Ensure that the number of tiles in each stripe is a multiple of the
392 - // groupsize; this avoids an annoying special case where a stripe starts
393 - // in the middle of group.
394 - iters = (group_blocks / thread_k_blocks) *
395 - ceildiv(iters, (group_blocks / thread_k_blocks));
396 - }
397 - }
398 -
399 - int slice_row = (iters * blockIdx.x) % k_tiles;
400 - int slice_col_par = (iters * blockIdx.x) / k_tiles;
401 - int slice_col = slice_col_par;
402 - int slice_iters; // number of threadblock tiles in the current slice
403 - int slice_count =
404 - 0; // total number of active threadblocks in the current slice
405 - int slice_idx; // index of threadblock in current slice; numbered bottom to
406 - // top
407 -
408 - // We can easily implement parallel problem execution by just remapping
409 - // indices and advancing global pointers
410 - if (slice_col_par >= n_tiles) {
411 - locks += (slice_col_par / n_tiles) * n_tiles;
412 - slice_col = slice_col_par % n_tiles;
413 - sorted_ids += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
414 - }
415 -
416 - // Compute all information about the current slice which is required for
417 - // synchronization.
418 - auto init_slice = [&]() {
419 - slice_iters =
420 - iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
421 - if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
422 - if (slice_iters == 0) return;
423 - if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
424 - slice_count = 1;
425 - slice_idx = 0;
426 - int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
427 - if (col_first <= k_tiles * (slice_col_par + 1)) {
428 - int col_off = col_first - k_tiles * slice_col_par;
429 - slice_count = ceildiv(k_tiles - col_off, iters);
430 - if (col_off > 0) slice_count++;
431 - int delta_first = iters * blockIdx.x - col_first;
432 - if (delta_first < 0 || (col_off == 0 && delta_first == 0))
433 - slice_idx = slice_count - 1;
434 - else {
435 - slice_idx = slice_count - 1 - delta_first / iters;
436 - if (col_off > 0) slice_idx--;
437 - }
438 - }
439 - if (slice_col == n_tiles) {
440 - sorted_ids += 16 * thread_m_blocks;
441 - locks += n_tiles;
442 - slice_col = 0;
443 - }
444 - };
445 - init_slice();
446 -
447 - // A sizes/strides
448 -
449 - // stride of the A matrix in global memory
450 - int a_gl_stride = prob_k / 8;
451 - // stride of an A matrix tile in shared memory
452 - constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
453 - // delta between subsequent A tiles in global memory
454 - constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
455 - // between subsequent accesses within a tile
456 - int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
457 - // between shared memory writes
458 - constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
459 - // between shared memory tile reads
460 - constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
461 - // within a shared memory tile
462 - constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
463 - // overall size of a tile
464 - constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
465 - // number of shared write iterations for a tile
466 - constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
467 -
468 - // B sizes/strides
469 - int b_gl_stride = 16 * prob_n / (pack_factor * 4);
470 - constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
471 - constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
472 - constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
473 -
474 - int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
475 - int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
476 - constexpr int b_sh_wr_delta = threads * b_thread_vecs;
477 - constexpr int b_sh_rd_delta = threads * b_thread_vecs;
478 - constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
479 - constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
480 -
481 - // Scale sizes/strides without act_order
482 - int s_gl_stride = prob_n / 8;
483 - constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
484 - constexpr int s_tb_groups =
485 - !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
486 - ? thread_k_blocks / group_blocks
487 - : 1;
488 - constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
489 - int s_gl_rd_delta = s_gl_stride;
490 - // Scale size/strides with act_order
491 - constexpr int tb_k = 16 * thread_k_blocks;
492 - constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
493 - // constexpr int act_s_row_stride = 1;
494 - // int act_s_col_stride = act_s_row_stride * num_groups;
495 - int act_s_col_stride = 1;
496 - int act_s_col_warp_stride = act_s_col_stride * 8;
497 - int tb_n_warps = thread_n_blocks / 4;
498 - int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
499 -
500 - constexpr int sorted_sh_stride = threads;
501 - constexpr int sorted_gl_stride = threads;
502 -
503 - // Global A read index of current thread.
504 - int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
505 - (threadIdx.x % a_gl_rd_delta_o);
506 - a_gl_rd += a_gl_rd_delta_o * slice_row;
507 - // Shared write index of current thread.
508 - int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
509 - (threadIdx.x % a_gl_rd_delta_o);
510 - // Shared read index.
511 - int a_sh_rd =
512 - a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
513 - a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
514 -
515 - int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
516 - (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
517 - b_gl_rd += b_sh_stride * slice_col;
518 - b_gl_rd += b_gl_rd_delta_o * slice_row;
519 - int b_sh_wr = threadIdx.x * b_thread_vecs;
520 - int b_sh_rd = threadIdx.x * b_thread_vecs;
521 -
522 - // For act_order
523 - constexpr int k_iter_size = tb_k / b_sh_wr_iters;
524 - int slice_k_start = tb_k * slice_row;
525 - int slice_k_finish = slice_k_start + tb_k * slice_iters;
526 - int slice_k_start_shared_fetch = slice_k_start;
527 - int slice_n_offset = act_s_col_tb_stride * slice_col;
528 -
529 - // No act_order
530 - int s_gl_rd;
531 - if constexpr (!has_act_order) {
532 - if constexpr (group_blocks == -1) {
533 - s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
534 - } else {
535 - s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
536 - s_sh_stride * slice_col + threadIdx.x;
537 - }
538 - }
539 - int s_sh_wr = threadIdx.x;
540 - bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
541 -
542 - // We use a different scale layout for grouped and column-wise quantization as
543 - // we scale a `half2` tile in column-major layout in the former and in
544 - // row-major in the latter case.
545 - int s_sh_rd;
546 - if constexpr (group_blocks != -1)
547 - s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
548 - (threadIdx.x % 32) / 4;
549 - else
550 - s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
551 - (threadIdx.x % 32) % 4;
552 -
553 - int sh_first_group_id = -1;
554 - int sh_num_groups = -1;
555 - constexpr int sh_max_num_groups = 32;
556 -
557 - int shs_size;
558 - if constexpr (has_act_order)
559 - shs_size = sh_max_num_groups * s_sh_stride + threads;
560 - else
561 - shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
562 -
563 - extern __shared__ int4 sh[];
564 - // Shared memory storage for global fetch pipelines.
565 - int4* sh_a = sh;
566 - int4* sh_b = sh_a + (stages * a_sh_stage);
567 - int4* sh_g_idx = sh_b + (stages * b_sh_stage);
568 - int4* sh_s = sh_g_idx + (stages * g_idx_stage);
569 - int* sh_sorted = (int*)(sh_s + shs_size);
570 -
571 - // Precompute which thread should not read memory in which iterations; this is
572 - // needed if there are more threads than required for a certain tilesize or
573 - // when the batchsize is not a multiple of 16.
574 - bool a_sh_wr_pred[a_sh_wr_iters];
575 - #pragma unroll
576 - for (int i = 0; i < a_sh_wr_iters; i++) {
577 - int a_idx = a_sh_wr_delta * i + a_sh_wr;
578 - int row = a_idx / a_gl_rd_delta_o;
579 - if (row >= prob_m) {
580 - a_sh_wr_pred[i] = false;
581 - } else {
582 - a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
583 - }
584 - }
585 -
586 - // To ensure that writing and reading A tiles to/from shared memory, the
587 - // latter in fragment format, is fully bank conflict free, we need to use a
588 - // rather fancy XOR-based layout. The key here is that neither reads nor
589 - // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
590 - // same shared memory banks. Further, it seems (based on NSight-Compute) that
591 - // each warp must also write a consecutive memory segment?
592 - auto transform_a = [&](int i) {
593 - int row = i / a_gl_rd_delta_o;
594 - return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
595 - };
596 - // Since the computation of this remapping is non-trivial and, due to our main
597 - // loop unrolls, all shared memory accesses are static, we simply precompute
598 - // both transformed reads and writes.
599 - int a_sh_wr_trans[a_sh_wr_iters];
600 - #pragma unroll
601 - for (int i = 0; i < a_sh_wr_iters; i++)
602 - a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
603 - int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
604 - #pragma unroll
605 - for (int i = 0; i < b_sh_wr_iters; i++) {
606 - #pragma unroll
607 - for (int j = 0; j < thread_m_blocks; j++)
608 - a_sh_rd_trans[i][j] =
609 - transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
610 - }
611 -
612 - // Since B-accesses have non-constant stride they have to be computed at
613 - // runtime; we break dependencies between subsequent accesses with a tile by
614 - // maintining multiple pointers (we have enough registers), a tiny
615 - // optimization.
616 - const int4* B_ptr[b_sh_wr_iters];
617 - #pragma unroll
618 - for (int i = 0; i < b_sh_wr_iters; i++)
619 - B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
620 -
621 - // Register storage for double buffer of shared memory reads.
622 - FragA frag_a[2][thread_m_blocks];
623 - I4 frag_b_quant[2][b_thread_vecs];
624 - FragC frag_c[thread_m_blocks][4][2];
625 - FragS frag_s[2][4]; // No act-order
626 - FragS act_frag_s[2][4][4]; // For act-order
627 -
628 - // Zero accumulators.
629 - auto zero_accums = [&]() {
630 - #pragma unroll
631 - for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
632 - reinterpret_cast<float*>(frag_c)[i] = 0;
633 - };
634 -
635 - auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
636 - int last_group_id) {
637 - sh_first_group_id = first_group_id;
638 - sh_num_groups = last_group_id - first_group_id + 1;
639 -
640 - if (sh_num_groups < sh_max_num_groups) {
641 - sh_num_groups = sh_max_num_groups;
642 - }
643 -
644 - if (sh_first_group_id + sh_num_groups > num_groups) {
645 - sh_num_groups = num_groups - sh_first_group_id;
646 - }
647 -
648 - int row_offset = first_group_id * s_gl_stride;
649 -
650 - if (is_async) {
651 - for (int i = 0; i < sh_num_groups; i++) {
652 - if (threadIdx.x < s_sh_stride) {
653 - cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
654 - &scales_ptr[row_offset + (i * s_gl_stride) +
655 - slice_n_offset + threadIdx.x]);
656 - }
657 - }
658 - } else {
659 - for (int i = 0; i < sh_num_groups; i++) {
660 - if (threadIdx.x < s_sh_stride) {
661 - sh_s[(i * s_sh_stride) + threadIdx.x] =
662 - scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
663 - threadIdx.x];
664 - }
665 - }
666 - }
667 - };
668 - // Asynchronously fetch the next A, B and s tile from global to the next
669 - // shared memory pipeline location.
670 - auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
671 - if (pred) {
672 - int4* sh_a_stage = sh_a + a_sh_stage * pipe;
673 - #pragma unroll
674 - for (int i = 0; i < a_sh_wr_iters; i++) {
675 - int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off;
676 - int row = a_idx / a_gl_stride;
677 - int sorted_row =
678 - replicate_input ? sorted_ids[row] / topk : sorted_ids[row];
679 - int new_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride;
680 - if (sorted_row < tot_m * (replicate_input ? 1 : topk) &&
681 - new_idx < a_gl_stride * tot_m * (replicate_input ? 1 : topk)) {
682 - cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[new_idx],
683 - a_sh_wr_pred[i]);
684 - }
685 - }
686 - int4* sh_b_stage = sh_b + b_sh_stage * pipe;
687 - #pragma unroll
688 - for (int i = 0; i < b_sh_wr_iters; i++) {
689 - #pragma unroll
690 - for (int j = 0; j < b_thread_vecs; j++) {
691 - cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
692 - }
693 - B_ptr[i] += b_gl_rd_delta_o;
694 - }
695 -
696 - if constexpr (has_act_order) {
697 - // Fetch g_idx thread-block portion
698 - int full_pipe = a_off;
699 - int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
700 - if (cur_k < prob_k && cur_k < slice_k_finish) {
701 - int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
702 -
703 - int4 const* cur_g_idx_stage_ptr =
704 - reinterpret_cast<int4 const*>(&g_idx[cur_k]);
705 -
706 - if (threadIdx.x < g_idx_stage) {
707 - cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
708 - &cur_g_idx_stage_ptr[threadIdx.x]);
709 - }
710 - }
711 - } else {
712 - if constexpr (group_blocks != -1) {
713 - int4* sh_s_stage = sh_s + s_sh_stage * pipe;
714 -
715 - if constexpr (group_blocks >= thread_k_blocks) {
716 - // Only fetch scales if this tile starts a new group
717 - if (pipe % (group_blocks / thread_k_blocks) == 0) {
718 - if (s_sh_wr_pred) {
719 - cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
720 - }
721 - s_gl_rd += s_gl_rd_delta;
722 - }
723 - } else {
724 - for (int i = 0; i < s_tb_groups; i++) {
725 - if (s_sh_wr_pred) {
726 - cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
727 - &scales_ptr[s_gl_rd]);
728 - }
729 - s_gl_rd += s_gl_rd_delta;
730 - }
731 - }
732 - }
733 - }
734 - }
735 - // Insert a fence even when we are winding down the pipeline to ensure that
736 - // waiting is also correct at this point.
737 - cp_async_fence();
738 - };
739 -
740 - // TODO we are currently hitting illegal memory accesses when fetching
741 - // sorted_ids to shared data: fix this
742 - auto fetch_sorted_ids_to_shared = [&]() {
743 - const int mpt = ceildiv(prob_m, threads);
744 - for (int i = 0; i < mpt; i++) {
745 - if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
746 - sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
747 - sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
748 - }
749 - }
750 - };
751 -
752 - // Wait until the next thread tile has been loaded to shared memory.
753 - auto wait_for_stage = [&]() {
754 - // We only have `stages - 2` active fetches since we are double buffering
755 - // and can only issue the next fetch when it is guaranteed that the previous
756 - // shared memory load is fully complete (as it may otherwise be
757 - // overwritten).
758 - cp_async_wait<stages - 2>();
759 - __syncthreads();
760 - };
761 -
762 - // Load the next sub-tile from the current location in the shared memory pipe
763 - // into the current register buffer.
764 - auto fetch_to_registers = [&](int k, int pipe) {
765 - int4* sh_a_stage = sh_a + a_sh_stage * pipe;
766 - #pragma unroll
767 - for (int i = 0; i < thread_m_blocks; i++)
768 - ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
769 - int4* sh_b_stage = sh_b + b_sh_stage * pipe;
770 -
771 - #pragma unroll
772 - for (int i = 0; i < b_thread_vecs; i++) {
773 - frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
774 - &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
775 - }
776 - };
777 -
778 - bool is_same_group[stages];
779 - int same_group_id[stages];
780 -
781 - auto init_same_group = [&](int pipe) {
782 - if constexpr (!has_act_order) {
783 - is_same_group[pipe] = false;
784 - same_group_id[pipe] = 0;
785 - return;
786 - }
787 -
788 - int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
789 - int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
790 -
791 - int group_id_1 = sh_g_idx_int_ptr[0];
792 - int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
793 -
794 - is_same_group[pipe] = group_id_1 == group_id_2;
795 - same_group_id[pipe] = group_id_1;
796 - };
797 -
798 - auto fetch_scales_to_registers = [&](int k, int full_pipe) {
799 - int pipe = full_pipe % stages;
800 -
801 - if constexpr (!has_act_order) {
802 - // No act-order case
803 - if constexpr (group_blocks != -1) {
804 - if constexpr (group_blocks >= thread_k_blocks) {
805 - int4* sh_s_stage =
806 - sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
807 - (pipe / (group_blocks / thread_k_blocks)));
808 - reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
809 - } else {
810 - int warp_id = threadIdx.x / 32;
811 - int n_warps = thread_n_blocks / 4;
812 -
813 - int warp_row = warp_id / n_warps;
814 -
815 - int cur_k = warp_row * 16;
816 - cur_k += k_iter_size * (k % b_sh_wr_iters);
817 -
818 - int k_blocks = cur_k / 16;
819 - int cur_group_id = k_blocks / group_blocks;
820 -
821 - int4* sh_s_stage = sh_s + s_sh_stage * pipe;
822 -
823 - reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
824 - sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
825 - }
826 - }
827 -
828 - return;
829 - }
830 -
831 - // Act-order case
832 -
833 - // Determine K of the "current" thread-block
834 - int cur_k = slice_k_start + tb_k * full_pipe;
835 - if (cur_k >= prob_k || cur_k >= slice_k_finish) {
836 - return;
837 - }
838 -
839 - // Reset (to current thread-block) since we read g_idx portion from the
840 - // shared memory
841 - cur_k = 0;
842 -
843 - // Progress to current iteration
844 - cur_k += k_iter_size * (k % b_sh_wr_iters);
845 -
846 - // Determine "position" inside the thread-block (based on warp and
847 - // thread-id)
848 - int warp_id = threadIdx.x / 32;
849 - int n_warps =
850 - thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
851 -
852 - int warp_row = warp_id / n_warps;
853 - int warp_col = warp_id % n_warps;
854 -
855 - cur_k += warp_row * 16;
856 -
857 - int th_id = threadIdx.x % 32;
858 - cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
859 -
860 - int s_col_shift =
861 - /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
862 - (th_id / 4) * act_s_col_stride;
863 -
864 - if (is_same_group[pipe]) {
865 - if (k % 2 == 0) {
866 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
867 - sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
868 - s_col_shift];
869 - } else {
870 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
871 - *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
872 - }
873 -
874 - for (int i = 1; i < 4; i++) {
875 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
876 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
877 - }
878 - return;
879 - }
880 -
881 - int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
882 - int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
883 -
884 - constexpr int k_frag_offsets[4] = {0, 1, 8,
885 - 9}; // Tensor core offsets per thread
886 -
887 - #pragma unroll
888 - for (int i = 0; i < 4; i++) {
889 - int actual_k = cur_k + k_frag_offsets[i];
890 -
891 - int group_id = sh_g_idx_int_ptr[actual_k];
892 - int rel_group_id = group_id - sh_first_group_id;
893 -
894 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
895 - sh_s[rel_group_id * s_sh_stride + s_col_shift];
896 - }
897 - };
898 -
899 - // Execute the actual tensor core matmul of a sub-tile.
900 - auto matmul = [&](int k) {
901 - // We have the m dimension as the inner loop in order to encourage overlapping
902 - // dequantization and matmul operations.
903 - #pragma unroll
904 - for (int j = 0; j < 4; j++) {
905 - int b_quant_0, b_quant_1;
906 - if constexpr (w_type.size_bits() == 4) {
907 - b_quant_0 = frag_b_quant[k % 2][0][j];
908 - b_quant_1 = b_quant_0 >> 8;
909 - } else {
910 - static_assert(w_type.size_bits() == 8);
911 - int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
912 - b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
913 - b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
914 - }
915 -
916 - FragB frag_b0 = dequant<w_type_id>(b_quant_0);
917 - FragB frag_b1 = dequant<w_type_id>(b_quant_1);
918 -
919 - // Apply scale to frag_b0
920 - if constexpr (has_act_order) {
921 - scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
922 - act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
923 - } else {
924 - if constexpr (group_blocks != -1) {
925 - scale(frag_b0, frag_s[k % 2][j], 0);
926 - }
927 - }
928 -
929 - // Apply scale to frag_b1
930 - if constexpr (has_act_order) {
931 - scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
932 - act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
933 -
934 - } else {
935 - if constexpr (group_blocks != -1) {
936 - scale(frag_b1, frag_s[k % 2][j], 1);
937 - }
938 - }
939 -
940 - #pragma unroll
941 - for (int i = 0; i < thread_m_blocks; i++) {
942 - mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
943 - mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
944 - }
945 - }
946 - };
947 -
948 - // Since we slice across the k dimension of a tile in order to increase the
949 - // number of warps while keeping the n dimension of a tile reasonable, we have
950 - // multiple warps that accumulate their partial sums of the same output
951 - // location; which we have to reduce over in the end. We do in shared memory.
952 - auto thread_block_reduce = [&]() {
953 - constexpr int red_off = threads / b_sh_stride_threads / 2;
954 - if (red_off >= 1) {
955 - int red_idx = threadIdx.x / b_sh_stride_threads;
956 - constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
957 - constexpr int red_sh_delta = b_sh_stride_threads;
958 - int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
959 - (threadIdx.x % b_sh_stride_threads);
960 -
961 - // Parallel logarithmic shared memory reduction. We make sure to avoid any
962 - // unnecessary read or write iterations, e.g., for two warps we write only
963 - // once by warp 1 and read only once by warp 0.
964 -
965 - #pragma unroll
966 - for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
967 - #pragma unroll
968 - for (int i = red_off; i > 0; i /= 2) {
969 - if (i <= red_idx && red_idx < 2 * i) {
970 - #pragma unroll
971 - for (int j = 0; j < 4 * 2; j++) {
972 - int red_sh_wr =
973 - red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
974 - if (i < red_off) {
975 - float* c_rd =
976 - reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
977 - float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
978 - #pragma unroll
979 - for (int k = 0; k < 4; k++)
980 - reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
981 - c_rd[k] + c_wr[k];
982 - }
983 - sh[red_sh_wr] =
984 - reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
985 - }
986 - }
987 - __syncthreads();
988 - }
989 - if (red_idx == 0) {
990 - #pragma unroll
991 - for (int i = 0; i < 4 * 2; i++) {
992 - float* c_rd =
993 - reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
994 - #pragma unroll
995 - for (int j = 0; j < 4; j++)
996 - reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
997 - c_rd[j];
998 - }
999 - }
1000 - __syncthreads();
1001 - }
1002 - }
1003 - };
1004 -
1005 - // Since multiple threadblocks may process parts of the same column slice, we
1006 - // finally have to globally reduce over the results. As the striped
1007 - // partitioning minimizes the number of such reductions and our outputs are
1008 - // usually rather small, we perform this reduction serially in L2 cache.
1009 - auto global_reduce = [&](bool first = false, bool last = false) {
1010 - // We are very careful here to reduce directly in the output buffer to
1011 - // maximize L2 cache utilization in this step. To do this, we write out
1012 - // results in FP16 (but still reduce with FP32 compute).
1013 - constexpr int active_threads = 32 * thread_n_blocks / 4;
1014 - if (threadIdx.x < active_threads) {
1015 - int c_gl_stride = prob_n / 8;
1016 - int c_gl_wr_delta_o = 8 * c_gl_stride;
1017 - int c_gl_wr_delta_i = 4 * (active_threads / 32);
1018 - int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
1019 - 4 * (threadIdx.x / 32) + threadIdx.x % 4;
1020 - c_gl_wr += (2 * thread_n_blocks) * slice_col;
1021 - constexpr int c_sh_wr_delta = active_threads;
1022 - int c_sh_wr = threadIdx.x;
1023 -
1024 - int row = (threadIdx.x % 32) / 4;
1025 -
1026 - if (!first) {
1027 - // Interestingly, doing direct global accesses here really seems to mess up
1028 - // the compiler and lead to slowdowns, hence we also use async-copies even
1029 - // though these fetches are not actually asynchronous.
1030 - #pragma unroll
1031 - for (int i = 0; i < thread_m_blocks * 4; i++) {
1032 - int c_idx =
1033 - c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
1034 - int sorted_row = sorted_ids[c_idx / c_gl_stride];
1035 - int new_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride;
1036 - cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i], &C[new_idx],
1037 - sorted_row < tot_m * topk &&
1038 - (8 * (i / 2) + row < prob_m &&
1039 - (i < (thread_m_blocks - 1) * 4 ||
1040 - sorted_ids[8 * (i / 2) + row] < tot_m * topk)));
1041 - }
1042 - cp_async_fence();
1043 - cp_async_wait<0>();
1044 - }
1045 -
1046 - #pragma unroll
1047 - for (int i = 0; i < thread_m_blocks * 4; i++) {
1048 - if (8 * (i / 2) + row < prob_m &&
1049 - (i < (thread_m_blocks - 1) * 4 ||
1050 - sorted_ids[8 * (i / 2) + row] < tot_m * topk)) {
1051 - if (!first) {
1052 - int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
1053 - #pragma unroll
1054 - for (int j = 0; j < 2 * 4; j++) {
1055 - reinterpret_cast<float*>(
1056 - &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
1057 - __half2float(reinterpret_cast<__half*>(&c_red)[j]);
1058 - }
1059 - }
1060 - if (!last) {
1061 - int4 c;
1062 - #pragma unroll
1063 - for (int j = 0; j < 2 * 4; j++) {
1064 - reinterpret_cast<__half*>(&c)[j] =
1065 - __float2half(reinterpret_cast<float*>(
1066 - &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
1067 - }
1068 - int c_idx =
1069 - c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
1070 - int row = sorted_ids[c_idx / c_gl_stride];
1071 - if (row < tot_m * topk) {
1072 - int new_idx = row * c_gl_stride + c_idx % c_gl_stride;
1073 - C[new_idx] = c;
1074 - }
1075 - }
1076 - }
1077 - }
1078 - }
1079 - };
1080 -
1081 - // Write out the reduce final result in the correct layout. We only actually
1082 - // reshuffle matrix fragments in this step, the reduction above is performed
1083 - // in fragment layout.
1084 - auto write_result = [&]() {
1085 - int c_gl_stride = prob_n / 8;
1086 - constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
1087 - int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
1088 - constexpr int c_sh_rd_delta =
1089 - c_sh_stride * (threads / (2 * thread_n_blocks));
1090 -
1091 - int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1092 - (threadIdx.x % (2 * thread_n_blocks));
1093 - c_gl_wr += (2 * thread_n_blocks) * slice_col;
1094 - int c_sh_wr =
1095 - (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
1096 - c_sh_wr += 32 * (threadIdx.x / 32);
1097 - int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1098 - (threadIdx.x % (2 * thread_n_blocks));
1099 -
1100 - int c_gl_wr_end = c_gl_stride * prob_m;
1101 -
1102 - // We first reorder in shared memory to guarantee the most efficient final
1103 - // global write patterns
1104 - auto write = [&](int idx, float c0, float c1, FragS& s) {
1105 - half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1106 -
1107 - // For per-column quantization we finally apply the scale here (only for
1108 - // 4-bit)
1109 - if constexpr (!has_act_order && group_blocks == -1 &&
1110 - w_type.size_bits() == 4) {
1111 - res = __hmul2(res, s[0]);
1112 - }
1113 -
1114 - ((half2*)sh)[idx] = res;
1115 - };
1116 - if (threadIdx.x / 32 < thread_n_blocks / 4) {
1117 - #pragma unroll
1118 - for (int i = 0; i < thread_m_blocks; i++) {
1119 - #pragma unroll
1120 - for (int j = 0; j < 4; j++) {
1121 - int wr = c_sh_wr + 8 * j;
1122 - write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
1123 - frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
1124 - write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
1125 - frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
1126 - write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
1127 - frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
1128 - write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
1129 - frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
1130 - }
1131 - c_sh_wr += 16 * (4 * c_sh_stride);
1132 - }
1133 - }
1134 - __syncthreads();
1135 -
1136 - #pragma unroll
1137 - for (int i = 0;
1138 - i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1139 - i++) {
1140 - if (c_gl_wr < c_gl_wr_end) {
1141 - int row = sorted_ids[c_gl_wr / c_gl_stride];
1142 - if (row < tot_m * topk) {
1143 - int off = row * c_gl_stride + c_gl_wr % c_gl_stride;
1144 - if (!apply_weights) {
1145 - C[off] = sh[c_sh_rd];
1146 - } else {
1147 - __half* ctrg = reinterpret_cast<__half*>(&C[off]);
1148 - __half* csrc = reinterpret_cast<__half*>(&sh[c_sh_rd]);
1149 - for (int j = 0; j < 8; ++j) {
1150 - ctrg[j] = __float2half(topk_weights[row] * __half2float(csrc[j]));
1151 - }
1152 - }
1153 - c_gl_wr += c_gl_wr_delta;
1154 - c_sh_rd += c_sh_rd_delta;
1155 - }
1156 - }
1157 - }
1158 - };
1159 -
1160 - // Start global fetch and register load pipelines.
1161 - auto start_pipes = [&]() {
1162 - // TODO re-enable after fixing this function
1163 - // fetch_sorted_ids_to_shared();
1164 - // __syncthreads();
1165 -
1166 - #pragma unroll
1167 - for (int i = 0; i < stages - 1; i++) {
1168 - if (has_act_order && i == 0) {
1169 - int last_g_idx = slice_k_start + stages * tb_k * 2;
1170 - if (last_g_idx >= prob_k) {
1171 - last_g_idx = prob_k - 1;
1172 - }
1173 - fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1174 - }
1175 - fetch_to_shared(i, i, i < slice_iters);
1176 - }
1177 -
1178 - zero_accums();
1179 - wait_for_stage();
1180 - init_same_group(0);
1181 - fetch_to_registers(0, 0);
1182 - fetch_scales_to_registers(0, 0);
1183 - a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1184 - slice_k_start_shared_fetch += tb_k * (stages - 1);
1185 - };
1186 - if (slice_iters) {
1187 - start_pipes();
1188 - }
1189 -
1190 - // Main loop.
1191 - while (slice_iters) {
1192 - // We unroll over both the global fetch and the register load pipeline to
1193 - // ensure all shared memory accesses are static. Note that both pipelines
1194 - // have even length meaning that the next iteration will always start at
1195 - // index 0.
1196 - #pragma unroll
1197 - for (int pipe = 0; pipe < stages;) {
1198 - #pragma unroll
1199 - for (int k = 0; k < b_sh_wr_iters; k++) {
1200 - fetch_to_registers(k + 1, pipe % stages);
1201 - fetch_scales_to_registers(k + 1, pipe);
1202 - if (k == b_sh_wr_iters - 2) {
1203 - fetch_to_shared((pipe + stages - 1) % stages, pipe,
1204 - slice_iters >= stages);
1205 - pipe++;
1206 - wait_for_stage();
1207 - init_same_group(pipe % stages);
1208 - }
1209 - matmul(k);
1210 - }
1211 - slice_iters--;
1212 - if (slice_iters == 0) {
1213 - break;
1214 - }
1215 - }
1216 -
1217 - a_gl_rd += a_gl_rd_delta_o * stages;
1218 - slice_k_start += tb_k * stages;
1219 - slice_k_start_shared_fetch += tb_k * stages;
1220 -
1221 - if constexpr (has_act_order) {
1222 - int first_group_id = g_idx[slice_k_start];
1223 - int last_g_idx = slice_k_start + stages * tb_k * 2;
1224 - if (last_g_idx >= prob_k) {
1225 - last_g_idx = prob_k - 1;
1226 - }
1227 - int last_group_id = g_idx[last_g_idx];
1228 - if (last_group_id >= sh_first_group_id + sh_num_groups) {
1229 - fetch_scales_to_shared(false, first_group_id, last_group_id);
1230 - __syncthreads();
1231 - }
1232 - }
1233 -
1234 - // Process results and, if necessary, proceed to the next column slice.
1235 - // While this pattern may not be the most readable, other ways of writing
1236 - // the loop seemed to noticeably worse performance after compilation.
1237 - if (slice_iters == 0) {
1238 - cp_async_wait<0>();
1239 - bool last = slice_idx == slice_count - 1;
1240 - if constexpr (!has_act_order && group_blocks == -1) {
1241 - if constexpr (w_type.size_bits() == 8) {
1242 - if (s_sh_wr_pred) {
1243 - cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1244 - }
1245 - cp_async_fence();
1246 - } else {
1247 - // For 4-bit per-column scales, we only fetch them here in the
1248 - // final step before write-out
1249 - if (last) {
1250 - if (s_sh_wr_pred) {
1251 - cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1252 - }
1253 - cp_async_fence();
1254 - }
1255 - }
1256 - }
1257 -
1258 - thread_block_reduce();
1259 - if constexpr (!has_act_order && group_blocks == -1) {
1260 - if constexpr (w_type.size_bits() == 8) {
1261 - cp_async_wait<0>();
1262 - __syncthreads();
1263 - if (threadIdx.x / 32 < thread_n_blocks / 4) {
1264 - reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1265 - reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1266 - }
1267 -
1268 - } else {
1269 - if (last) {
1270 - cp_async_wait<0>();
1271 - __syncthreads();
1272 - if (threadIdx.x / 32 < thread_n_blocks / 4) {
1273 - reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1274 - reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1275 - }
1276 - }
1277 - }
1278 - }
1279 -
1280 - // For 8-bit channelwise, we apply the scale before the global reduction
1281 - // that converts the fp32 results to fp16 (so that we avoid possible
1282 - // overflow in fp16)
1283 - if constexpr (!has_act_order && group_blocks == -1 &&
1284 - w_type.size_bits() == 8) {
1285 - if (threadIdx.x / 32 < thread_n_blocks / 4) {
1286 - #pragma unroll
1287 - for (int i = 0; i < thread_m_blocks; i++) {
1288 - #pragma unroll
1289 - for (int j = 0; j < 4; j++) {
1290 - scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1291 - frag_s[j / 2][2 * (j % 2) + 0]);
1292 - scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1293 - frag_s[j / 2][2 * (j % 2) + 0]);
1294 -
1295 - scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1296 - frag_s[j / 2][2 * (j % 2) + 1]);
1297 - scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1298 - frag_s[j / 2][2 * (j % 2) + 1]);
1299 - }
1300 - }
1301 - }
1302 - }
1303 -
1304 - if (slice_count > 1) { // only globally reduce if there is more than one
1305 - // block in a slice
1306 - barrier_acquire(&locks[slice_col], slice_idx);
1307 - global_reduce(slice_idx == 0, last);
1308 - barrier_release(&locks[slice_col], last);
1309 - }
1310 - if (last) // only the last block in a slice actually writes the result
1311 - write_result();
1312 - slice_row = 0;
1313 - slice_col_par++;
1314 - slice_col++;
1315 - init_slice();
1316 - if (slice_iters) {
1317 - a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1318 - (threadIdx.x % a_gl_rd_delta_o);
1319 - #pragma unroll
1320 - for (int i = 0; i < b_sh_wr_iters; i++)
1321 - B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1322 - if (slice_col == 0) {
1323 - #pragma unroll
1324 - for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1325 - }
1326 -
1327 - // Update slice k/n for scales loading
1328 - if constexpr (has_act_order) {
1329 - slice_k_start = tb_k * slice_row;
1330 - slice_k_finish = slice_k_start + tb_k * slice_iters;
1331 - slice_k_start_shared_fetch = slice_k_start;
1332 - slice_n_offset = act_s_col_tb_stride * slice_col;
1333 -
1334 - } else {
1335 - s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1336 - }
1337 - start_pipes();
1338 - }
1339 - }
1340 - }
1341 -}
1342 -
1343 -template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1344 - const int threads, // number of threads in a threadblock
1345 - const int thread_n_blocks, // same for n dimension (output)
1346 - const int thread_k_blocks, // same for k dimension (reduction)
1347 - const int stages, // number of stages for the async global->shared
1348 - // fetch pipeline
1349 - const bool has_act_order, // whether act_order is enabled
1350 - const int group_blocks = -1 // number of consecutive 16x16 blocks
1351 - // with a separate quantization scale
1352 - >
1353 -__global__ void MarlinMoE(
1354 - const int4* __restrict__ A, // fp16 input matrix of shape mxk
1355 - const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1356 - int4* __restrict__ C, // fp16 output buffer of shape mxn
1357 - const int* __restrict__ sorted_ids_base, // int32 sorted ids of experts
1358 - const float* __restrict__ topk_weights, // float topk weights
1359 - const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1360 - // (k/groupsize)xn
1361 - const int* __restrict__ g_idx, // int32 group indices of shape k
1362 - const int* __restrict__ expert_offsets,
1363 - int num_groups, // number of scale groups per output channel
1364 - int expert_idx, // idx of current expert
1365 - int num_experts, // number of experts
1366 - int topk, // topk parameter of moe
1367 - int prob_m, // batch dimension m
1368 - int prob_n, // output dimension n
1369 - int prob_k, // reduction dimension k
1370 - int tot_m, // total number of rows in A and C
1371 - int* locks, // extra global storage for barrier synchronization
1372 - bool replicate_input, // do we use the same input for each expert?
1373 - bool apply_weights, // apply weights to output
1374 - int current_m_block, // current m block to start kernel computation from
1375 - int max_par, // maximum parallelism
1376 - int cfg_max_m_blocks // upper bound on m blocks
1377 -) {
1378 - int m_block_ctr = current_m_block;
1379 -
1380 - const int* sorted_ids_expert =
1381 - sorted_ids_base + expert_offsets[expert_idx] + m_block_ctr * 4 * max_par;
1382 - int tot_its = expert_offsets[expert_idx + 1] - expert_offsets[expert_idx];
1383 - if (tot_its == 0) {
1384 - return;
1385 - }
1386 - int tot_m_blocks = ceildiv(tot_its, 16);
1387 - int pad = 16 * tot_m_blocks - tot_its;
1388 -
1389 - if (m_block_ctr >= tot_m_blocks) {
1390 - return;
1391 - }
1392 -
1393 - int max_block = tot_m_blocks - m_block_ctr;
1394 - prob_m = tot_its - 16 * m_block_ctr;
1395 -
1396 - int par = 1;
1397 - if (max_block > cfg_max_m_blocks) {
1398 - // Note that parallel > 1 currently only works for inputs without any
1399 - // padding
1400 - par = (16 * max_block - pad) / (16 * cfg_max_m_blocks);
1401 - if (par > max_par) par = max_par;
1402 - prob_m = (16 * cfg_max_m_blocks) * par;
1403 - m_block_ctr += cfg_max_m_blocks * (par - 1);
1404 - max_block = cfg_max_m_blocks;
1405 - }
1406 -
1407 - if (max_block == 1) {
1408 - MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1409 - stages, has_act_order, group_blocks>(
1410 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1411 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1412 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1413 - current_m_block);
1414 - } else if (max_block == 2) {
1415 - MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1416 - stages, has_act_order, group_blocks>(
1417 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1418 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1419 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1420 - current_m_block);
1421 - } else if (max_block == 3) {
1422 - MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1423 - stages, has_act_order, group_blocks>(
1424 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1425 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1426 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1427 - current_m_block);
1428 - } else {
1429 - MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1430 - stages, has_act_order, group_blocks>(
1431 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1432 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1433 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1434 - current_m_block);
1435 - }
1436 -}
1437 -
118 #else
119
120 __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
@@ -1454,81 +134,8 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
134 return;
135 }
136
1457 -template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1458 - const int threads, // number of threads in a threadblock
1459 - const int thread_n_blocks, // same for n dimension (output)
1460 - const int thread_k_blocks, // same for k dimension (reduction)
1461 - const int stages, // number of stages for the async global->shared
1462 - // fetch pipeline
1463 - const bool has_act_order, // whether act_order is enabled
1464 - const int group_blocks = -1 // number of consecutive 16x16 blocks
1465 - // with a separate quantization scale
1466 - >
1467 -__global__ void MarlinMoE(
1468 - const int4* __restrict__ A, // fp16 input matrix of shape mxk
1469 - const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1470 - int4* __restrict__ C, // fp16 output buffer of shape mxn
1471 - const int* __restrict__ sorted_ids, // int32 sorted ids of experts
1472 - const float* __restrict__ topk_weights, // float topk weights
1473 - const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1474 - // (k/groupsize)xn
1475 - const int* __restrict__ g_idx, // int32 group indices of shape k
1476 - const int* __restrict__ expert_offsets,
1477 - int num_groups, // number of scale groups per output channel
1478 - int expert_idx, // idx of current expert
1479 - int num_experts, // number of experts
1480 - int topk, // topk parameter of moe
1481 - int prob_m, // batch dimension m
1482 - int prob_n, // output dimension n
1483 - int prob_k, // reduction dimension k
1484 - int tot_m, // total number of rows in A and C
1485 - int* locks, // extra global storage for barrier synchronization
1486 - bool replicate_input, // do we use the same input for each expert?
1487 - bool apply_weights, // apply weights to output
1488 - int current_m_block, // current m block to start kernel computation from
1489 - int max_par, // maximum parallelism
1490 - int cfg_max_m_blocks // upper bound on m blocks
1491 -
1492 -) {
1493 - // Marlin is not implemented yet for SM < 8.0
1494 - assert(false);
1495 - return;
1496 -}
1497 -
137 #endif
138
1500 -// 8 warps are a good choice since every SM has 4 schedulers and having more
1501 -// than 1 warp per schedule allows some more latency hiding. At the same time,
1502 -// we want relatively few warps to have many registers per warp and small tiles.
1503 -const int USER_THREADS =
1504 - 256; // Note: This is only used with user-provided thread_k/n
1505 -const int STAGES = 4; // 4 pipeline stages fit into shared memory
1506 -// const int SHARED_MEM =
1507 -// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1508 -
1509 -static constexpr int min_thread_n = 64;
1510 -static constexpr int min_thread_k = 64;
1511 -
1512 -#define __CALL_IF_MOE(W_TYPE, THREAD_N_BLOCKS, THREAD_K_BLOCKS, HAS_ACT_ORDER, \
1513 - GROUP_BLOCKS, NUM_THREADS) \
1514 - else if (q_type == W_TYPE && thread_n_blocks == THREAD_N_BLOCKS && \
1515 - thread_k_blocks == THREAD_K_BLOCKS && \
1516 - has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1517 - num_threads == NUM_THREADS) { \
1518 - cudaFuncSetAttribute( \
1519 - MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1520 - STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1521 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1522 - MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1523 - STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1524 - <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1525 - A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1526 - g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1527 - num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1528 - replicate_input, apply_weights, m_block, max_par, \
1529 - exec_cfg.max_m_blocks); \
1530 - }
1531 -
139 typedef struct {
140 int thread_k;
141 int thread_n;
@@ -1703,25 +310,27 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
310 return exec_config_t{0, {-1, -1, -1}};
311 }
312
1706 -#define CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1707 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1708 - \
1709 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1710 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1711 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1712 - __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1713 -
1714 -void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1715 - const void* sorted_ids, const void* topk_weights,
1716 - const void* topk_ids, const void* s, const void* g_idx,
1717 - const void* perm, void* a_tmp, void* expert_offsets,
1718 - int prob_m, int prob_n, int prob_k, void* workspace,
1719 - vllm::ScalarType const& q_type, bool has_act_order,
1720 - bool is_k_full, int num_groups, int group_size,
1721 - int num_experts, int topk, int moe_block_size, int dev,
1722 - cudaStream_t stream, int thread_k, int thread_n,
1723 - int sms, int max_par, bool replicate_input,
1724 - bool apply_weights) {
313 +#define CALL_MOE_KERNEL_FUNCTION(KERNEL_FUNCTION) \
314 + else if (KERNEL_FUNCTION(q_type, thread_n_blocks, thread_k_blocks, \
315 + has_act_order, group_blocks, num_threads, blocks, \
316 + max_shared_mem, stream, A_ptr, B_ptr, C_ptr, \
317 + sorted_ids_ptr, topk_weights_ptr, s_ptr, g_idx_ptr, \
318 + expert_offsets_ptr, num_groups, expert_idx, \
319 + num_experts, topk, prob_m, prob_n, prob_k, tot_m, \
320 + locks, replicate_input, apply_weights, m_block, \
321 + max_par, exec_cfg.max_m_blocks)) { \
322 + }
323 +
324 +void marlin_mm_moe(const void* A, const void* B, void* C,
325 + const void* sorted_ids, const void* topk_weights,
326 + const void* topk_ids, const void* s, const void* g_idx,
327 + const void* perm, void* a_tmp, void* expert_offsets,
328 + int prob_m, int prob_n, int prob_k, void* workspace,
329 + vllm::ScalarType const& q_type, bool has_act_order,
330 + bool is_k_full, int num_groups, int group_size,
331 + int num_experts, int topk, int moe_block_size, int dev,
332 + cudaStream_t stream, int thread_k, int thread_n, int sms,
333 + int max_par, bool replicate_input, bool apply_weights) {
334 TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
335 ", ", prob_n, ", ", prob_k, "]");
336
@@ -1845,26 +454,16 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
454 int tot_m_blocks = ceildiv(tot_m, 16);
455 for (int m_block = 0; m_block < tot_m_blocks;
456 m_block += 4 * exec_cfg.max_m_blocks) {
1848 - // make it max possible value
1849 - int thread_m_blocks = exec_cfg.max_m_blocks;
1850 -
457 if (false) {
458 }
1853 - CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
1854 - CALL_IF_MOE(vllm::kU4B8, 8, 8, 256)
1855 - CALL_IF_MOE(vllm::kU4B8, 8, 4, 128)
1856 - CALL_IF_MOE(vllm::kU4B8, 4, 8, 128)
1857 - CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
1858 - CALL_IF_MOE(vllm::kU8B128, 8, 8, 256)
1859 - CALL_IF_MOE(vllm::kU8B128, 8, 4, 128)
1860 - CALL_IF_MOE(vllm::kU8B128, 4, 8, 128)
459 + CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku4b8)
460 + CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku8b128)
461 else {
462 TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
463 str(prob_n) + ", " + str(prob_k) + "]" +
464 ", has_act_order = " + str(has_act_order) +
465 ", num_groups = " + str(num_groups) +
466 ", group_size = " + str(group_size) +
1867 - ", thread_m_blocks = " + str(thread_m_blocks) +
467 ", thread_n_blocks = " + str(thread_n_blocks) +
468 ", thread_k_blocks = " + str(thread_k_blocks));
469 }
@@ -1943,7 +542,7 @@ torch::Tensor marlin_gemm_moe(
542 }
543 }
544
1946 - marlin_moe::marlin_mm_moe_f16i4(
545 + marlin_moe::marlin_mm_moe(
546 a.data_ptr(), b_q_weights.data_ptr(), c.data_ptr(), sorted_ids.data_ptr(),
547 topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
548 g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),