A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Kernel] Split Marlin MoE kernels into multiple files (#8661)
Co-authored-by: mgoin <michael@neuralmagic.com>
ElizaWszola committed
Sep 24, 2024 at 18:31 UTC
a928ded99519f803d4cf6389df6acc707239a5cc
7 files changed
+1552
-1427
CMakeLists.txt
+5
@@ -316,6 +316,11 @@ set(VLLM_MOE_EXT_SRC
316
317
if(VLLM_GPU_LANG STREQUAL "CUDA")
318
list(APPEND VLLM_MOE_EXT_SRC
319
+ "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
320
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h"
321
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
322
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
323
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
324
"csrc/moe/marlin_moe_ops.cu")
325
endif()
326
csrc/moe/marlin_kernels/marlin_moe_kernel.h
new
+1425
@@ -0,0 +1,1425 @@
1
+#pragma once
2
+
3
+#include <torch/all.h>
4
+
5
+#include <ATen/cuda/CUDAContext.h>
6
+#include <c10/cuda/CUDAGuard.h>
7
+#include <cuda.h>
8
+#include <cuda_fp16.h>
9
+#include <cuda_runtime.h>
10
+
11
+#include <iostream>
12
+
13
+#include "core/scalar_type.hpp"
14
+
15
+namespace marlin_moe {
16
+
17
+constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
18
+
19
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
20
+
21
+// Instances of `Vec` are used to organize groups of >>registers<<, as needed
22
+// for instance as inputs to tensor core operations. Consequently, all
23
+// corresponding index accesses must be compile-time constants, which is why we
24
+// extensively use `#pragma unroll` throughout the kernel code to guarantee
25
+// this.
26
+template <typename T, int n>
27
+struct Vec {
28
+ T elems[n];
29
+ __device__ T& operator[](int i) { return elems[i]; }
30
+};
31
+
32
+using I4 = Vec<int, 4>;
33
+
34
+// Matrix fragments for tensor core instructions; their precise layout is
35
+// documented here:
36
+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
37
+using FragA = Vec<half2, 4>;
38
+using FragB = Vec<half2, 2>;
39
+using FragC = Vec<float, 4>;
40
+using FragS = Vec<half2, 1>; // quantization scales
41
+
42
+// Predicated asynchronous global->shared copy; used for inputs A where we apply
43
+// predication to handle batchsizes that are not multiples of 16.
44
+__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
45
+ bool pred = true) {
46
+ const int BYTES = 16;
47
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
48
+ asm volatile(
49
+ "{\n"
50
+ " .reg .pred p;\n"
51
+ " setp.ne.b32 p, %0, 0;\n"
52
+ " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
53
+ "}\n" ::"r"((int)pred),
54
+ "r"(smem), "l"(glob_ptr), "n"(BYTES));
55
+}
56
+
57
+// Asynchronous global->shared copy
58
+__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
59
+ const int BYTES = 16;
60
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
61
+ asm volatile(
62
+ "{\n"
63
+ " cp.async.cg.shared.global [%0], [%1], %2;\n"
64
+ "}\n" ::"r"(smem),
65
+ "l"(glob_ptr), "n"(BYTES));
66
+}
67
+
68
+// Async copy fence.
69
+__device__ inline void cp_async_fence() {
70
+ asm volatile("cp.async.commit_group;\n" ::);
71
+}
72
+
73
+// Wait until at most `n` async copy stages are still pending.
74
+template <int n>
75
+__device__ inline void cp_async_wait() {
76
+ asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
77
+}
78
+
79
+// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
80
+// output/accumulation.
81
+__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
82
+ FragC& frag_c) {
83
+ const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
84
+ const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
85
+ float* c = reinterpret_cast<float*>(&frag_c);
86
+ asm volatile(
87
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
88
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
89
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
90
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
91
+ "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
92
+}
93
+
94
+// Instruction for loading a full 16x16 matrix fragment of operand A from shared
95
+// memory, directly in tensor core layout.
96
+__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
97
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
98
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
99
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
100
+ : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
101
+ : "r"(smem));
102
+}
103
+
104
+// Lookup-table based 3-input logical operation; explicitly used for
105
+// dequantization as the compiler does not seem to automatically recognize it in
106
+// all cases.
107
+template <int lut>
108
+__device__ inline int lop3(int a, int b, int c) {
109
+ int res;
110
+ asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
111
+ : "=r"(res)
112
+ : "r"(a), "r"(b), "r"(c), "n"(lut));
113
+ return res;
114
+}
115
+
116
+// Constructs destination register by taking bytes from 2 sources (based on
117
+// mask)
118
+template <int start_byte, int mask>
119
+__device__ inline uint32_t prmt(uint32_t a) {
120
+ uint32_t res;
121
+ asm volatile("prmt.b32 %0, %1, %2, %3;\n"
122
+ : "=r"(res)
123
+ : "r"(a), "n"(start_byte), "n"(mask));
124
+ return res;
125
+}
126
+
127
+template <vllm::ScalarTypeId w_type_id>
128
+__device__ inline FragB dequant(int q);
129
+
130
+// Efficiently dequantize 4bit values packed in an int32 value into a full
131
+// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
132
+// with some small changes:
133
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
134
+template <>
135
+__device__ inline FragB dequant<vllm::kU4B8.id()>(int q) {
136
+ const int LO = 0x000f000f;
137
+ const int HI = 0x00f000f0;
138
+ const int EX = 0x64006400;
139
+ // Guarantee that the `(a & b) | c` operations are LOP3s.
140
+ int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
141
+ int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
142
+ // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
143
+ // directly into `SUB` and `ADD`.
144
+ const int SUB = 0x64086408;
145
+ const int MUL = 0x2c002c00;
146
+ const int ADD = 0xd480d480;
147
+ FragB frag_b;
148
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
149
+ *reinterpret_cast<const half2*>(&SUB));
150
+ frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
151
+ *reinterpret_cast<const half2*>(&MUL),
152
+ *reinterpret_cast<const half2*>(&ADD));
153
+ return frag_b;
154
+}
155
+
156
+// Fast Int8ToFp16: Efficiently dequantize 8bit int values to fp16
157
+// Reference:
158
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
159
+template <>
160
+__device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
161
+ static constexpr uint32_t mask_for_elt_01 = 0x5250;
162
+ static constexpr uint32_t mask_for_elt_23 = 0x5351;
163
+ static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
164
+
165
+ uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
166
+ uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
167
+
168
+ static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
169
+
170
+ FragB frag_b;
171
+ frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
172
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
173
+ frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
174
+ *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
175
+ return frag_b;
176
+}
177
+
178
+// Multiply dequantized values by the corresponding quantization scale; used
179
+// only for grouped quantization.
180
+__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
181
+ half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
182
+ frag_b[0] = __hmul2(frag_b[0], s);
183
+ frag_b[1] = __hmul2(frag_b[1], s);
184
+}
185
+
186
+// Given 2 floats multiply by 2 scales (halves)
187
+__device__ inline void scale_float(float* c, FragS& s) {
188
+ __half* s_ptr = reinterpret_cast<__half*>(&s);
189
+ c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
190
+ c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
191
+}
192
+
193
+// Same as above, but for act_order (each K is multiplied individually)
194
+__device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
195
+ FragS& frag_s_3, FragS& frag_s_4, int i) {
196
+ __half2 s_val_1_2;
197
+ s_val_1_2.x = reinterpret_cast<__half*>(&frag_s_1)[i];
198
+ s_val_1_2.y = reinterpret_cast<__half*>(&frag_s_2)[i];
199
+
200
+ __half2 s_val_3_4;
201
+ s_val_3_4.x = reinterpret_cast<__half*>(&frag_s_3)[i];
202
+ s_val_3_4.y = reinterpret_cast<__half*>(&frag_s_4)[i];
203
+
204
+ frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
205
+ frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
206
+}
207
+
208
+// Wait until barrier reaches `count`, then lock for current threadblock.
209
+__device__ inline void barrier_acquire(int* lock, int count) {
210
+ if (threadIdx.x == 0) {
211
+ int state = -1;
212
+ do
213
+ // Guarantee that subsequent writes by this threadblock will be visible
214
+ // globally.
215
+ asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
216
+ : "=r"(state)
217
+ : "l"(lock));
218
+ while (state != count);
219
+ }
220
+ __syncthreads();
221
+}
222
+
223
+// Release barrier and increment visitation count.
224
+__device__ inline void barrier_release(int* lock, bool reset = false) {
225
+ __syncthreads();
226
+ if (threadIdx.x == 0) {
227
+ if (reset) {
228
+ lock[0] = 0;
229
+ return;
230
+ }
231
+ int val = 1;
232
+ // Make sure that all writes since acquiring this barrier are visible
233
+ // globally, while releasing the barrier.
234
+ asm volatile("fence.acq_rel.gpu;\n");
235
+ asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
236
+ :
237
+ : "l"(lock), "r"(val));
238
+ }
239
+}
240
+
241
+template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
242
+ const int threads, // number of threads in a threadblock
243
+ const int thread_m_blocks, // number of 16x16 blocks in the m
244
+ // dimension (batchsize) of the
245
+ // threadblock
246
+ const int thread_n_blocks, // same for n dimension (output)
247
+ const int thread_k_blocks, // same for k dimension (reduction)
248
+ const int stages, // number of stages for the async global->shared
249
+ // fetch pipeline
250
+ const bool has_act_order, // whether act_order is enabled
251
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
252
+ // with a separate quantization scale
253
+ >
254
+__device__ inline void MarlinMoESingle(
255
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
256
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
257
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
258
+ const int* __restrict__ sorted_ids, // int32 sorted ids of experts
259
+ const float* __restrict__ topk_weights, // float topk weights
260
+ const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
261
+ // (k/groupsize)xn
262
+ const int* __restrict__ g_idx, // int32 group indices of shape k
263
+ const int* __restrict__ expert_offsets,
264
+ int num_groups, // number of scale groups per output channel
265
+ int expert_idx, // idx of current expert
266
+ int num_experts, // number of experts
267
+ int topk, // topk parameter of moe
268
+ int prob_m, // batch dimension m
269
+ int prob_n, // output dimension n
270
+ int prob_k, // reduction dimension k
271
+ int tot_m, // total number of rows in A and C
272
+ int* locks, // extra global storage for barrier synchronization
273
+ bool replicate_input, // do we use the same input for each expert?
274
+ bool apply_weights, // apply weights to output
275
+ int current_m_block // current m block to start kernel computation from
276
+) {
277
+ static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
278
+ constexpr int pack_factor = 32 / w_type.size_bits();
279
+
280
+ // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
281
+ // better partitioning with less reductions
282
+ int parallel = 1;
283
+ if (prob_m > 16 * thread_m_blocks) {
284
+ parallel = prob_m / (16 * thread_m_blocks);
285
+ prob_m = 16 * thread_m_blocks;
286
+ }
287
+
288
+ int k_tiles = prob_k / 16 / thread_k_blocks;
289
+ int n_tiles = prob_n / 16 / thread_n_blocks;
290
+ int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
291
+
292
+ if constexpr (!has_act_order && group_blocks != -1) {
293
+ if (group_blocks >= thread_k_blocks) {
294
+ // Ensure that the number of tiles in each stripe is a multiple of the
295
+ // groupsize; this avoids an annoying special case where a stripe starts
296
+ // in the middle of group.
297
+ iters = (group_blocks / thread_k_blocks) *
298
+ ceildiv(iters, (group_blocks / thread_k_blocks));
299
+ }
300
+ }
301
+
302
+ int slice_row = (iters * blockIdx.x) % k_tiles;
303
+ int slice_col_par = (iters * blockIdx.x) / k_tiles;
304
+ int slice_col = slice_col_par;
305
+ int slice_iters; // number of threadblock tiles in the current slice
306
+ int slice_count =
307
+ 0; // total number of active threadblocks in the current slice
308
+ int slice_idx; // index of threadblock in current slice; numbered bottom to
309
+ // top
310
+
311
+ // We can easily implement parallel problem execution by just remapping
312
+ // indices and advancing global pointers
313
+ if (slice_col_par >= n_tiles) {
314
+ locks += (slice_col_par / n_tiles) * n_tiles;
315
+ slice_col = slice_col_par % n_tiles;
316
+ sorted_ids += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
317
+ }
318
+
319
+ // Compute all information about the current slice which is required for
320
+ // synchronization.
321
+ auto init_slice = [&]() {
322
+ slice_iters =
323
+ iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
324
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
325
+ if (slice_iters == 0) return;
326
+ if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
327
+ slice_count = 1;
328
+ slice_idx = 0;
329
+ int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
330
+ if (col_first <= k_tiles * (slice_col_par + 1)) {
331
+ int col_off = col_first - k_tiles * slice_col_par;
332
+ slice_count = ceildiv(k_tiles - col_off, iters);
333
+ if (col_off > 0) slice_count++;
334
+ int delta_first = iters * blockIdx.x - col_first;
335
+ if (delta_first < 0 || (col_off == 0 && delta_first == 0))
336
+ slice_idx = slice_count - 1;
337
+ else {
338
+ slice_idx = slice_count - 1 - delta_first / iters;
339
+ if (col_off > 0) slice_idx--;
340
+ }
341
+ }
342
+ if (slice_col == n_tiles) {
343
+ sorted_ids += 16 * thread_m_blocks;
344
+ locks += n_tiles;
345
+ slice_col = 0;
346
+ }
347
+ };
348
+ init_slice();
349
+
350
+ // A sizes/strides
351
+
352
+ // stride of the A matrix in global memory
353
+ int a_gl_stride = prob_k / 8;
354
+ // stride of an A matrix tile in shared memory
355
+ constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
356
+ // delta between subsequent A tiles in global memory
357
+ constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
358
+ // between subsequent accesses within a tile
359
+ int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
360
+ // between shared memory writes
361
+ constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
362
+ // between shared memory tile reads
363
+ constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
364
+ // within a shared memory tile
365
+ constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
366
+ // overall size of a tile
367
+ constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
368
+ // number of shared write iterations for a tile
369
+ constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
370
+
371
+ // B sizes/strides
372
+ int b_gl_stride = 16 * prob_n / (pack_factor * 4);
373
+ constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
374
+ constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
375
+ constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
376
+
377
+ int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
378
+ int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
379
+ constexpr int b_sh_wr_delta = threads * b_thread_vecs;
380
+ constexpr int b_sh_rd_delta = threads * b_thread_vecs;
381
+ constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
382
+ constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
383
+
384
+ // Scale sizes/strides without act_order
385
+ int s_gl_stride = prob_n / 8;
386
+ constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
387
+ constexpr int s_tb_groups =
388
+ !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
389
+ ? thread_k_blocks / group_blocks
390
+ : 1;
391
+ constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
392
+ int s_gl_rd_delta = s_gl_stride;
393
+ // Scale size/strides with act_order
394
+ constexpr int tb_k = 16 * thread_k_blocks;
395
+ constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
396
+ // constexpr int act_s_row_stride = 1;
397
+ // int act_s_col_stride = act_s_row_stride * num_groups;
398
+ int act_s_col_stride = 1;
399
+ int act_s_col_warp_stride = act_s_col_stride * 8;
400
+ int tb_n_warps = thread_n_blocks / 4;
401
+ int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
402
+
403
+ constexpr int sorted_sh_stride = threads;
404
+ constexpr int sorted_gl_stride = threads;
405
+
406
+ // Global A read index of current thread.
407
+ int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
408
+ (threadIdx.x % a_gl_rd_delta_o);
409
+ a_gl_rd += a_gl_rd_delta_o * slice_row;
410
+ // Shared write index of current thread.
411
+ int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
412
+ (threadIdx.x % a_gl_rd_delta_o);
413
+ // Shared read index.
414
+ int a_sh_rd =
415
+ a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
416
+ a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
417
+
418
+ int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
419
+ (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
420
+ b_gl_rd += b_sh_stride * slice_col;
421
+ b_gl_rd += b_gl_rd_delta_o * slice_row;
422
+ int b_sh_wr = threadIdx.x * b_thread_vecs;
423
+ int b_sh_rd = threadIdx.x * b_thread_vecs;
424
+
425
+ // For act_order
426
+ constexpr int k_iter_size = tb_k / b_sh_wr_iters;
427
+ int slice_k_start = tb_k * slice_row;
428
+ int slice_k_finish = slice_k_start + tb_k * slice_iters;
429
+ int slice_k_start_shared_fetch = slice_k_start;
430
+ int slice_n_offset = act_s_col_tb_stride * slice_col;
431
+
432
+ // No act_order
433
+ int s_gl_rd;
434
+ if constexpr (!has_act_order) {
435
+ if constexpr (group_blocks == -1) {
436
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
437
+ } else {
438
+ s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
439
+ s_sh_stride * slice_col + threadIdx.x;
440
+ }
441
+ }
442
+ int s_sh_wr = threadIdx.x;
443
+ bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
444
+
445
+ // We use a different scale layout for grouped and column-wise quantization as
446
+ // we scale a `half2` tile in column-major layout in the former and in
447
+ // row-major in the latter case.
448
+ int s_sh_rd;
449
+ if constexpr (group_blocks != -1)
450
+ s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
451
+ (threadIdx.x % 32) / 4;
452
+ else
453
+ s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
454
+ (threadIdx.x % 32) % 4;
455
+
456
+ int sh_first_group_id = -1;
457
+ int sh_num_groups = -1;
458
+ constexpr int sh_max_num_groups = 32;
459
+
460
+ int shs_size;
461
+ if constexpr (has_act_order)
462
+ shs_size = sh_max_num_groups * s_sh_stride + threads;
463
+ else
464
+ shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
465
+
466
+ extern __shared__ int4 sh[];
467
+ // Shared memory storage for global fetch pipelines.
468
+ int4* sh_a = sh;
469
+ int4* sh_b = sh_a + (stages * a_sh_stage);
470
+ int4* sh_g_idx = sh_b + (stages * b_sh_stage);
471
+ int4* sh_s = sh_g_idx + (stages * g_idx_stage);
472
+ int* sh_sorted = (int*)(sh_s + shs_size);
473
+
474
+ // Precompute which thread should not read memory in which iterations; this is
475
+ // needed if there are more threads than required for a certain tilesize or
476
+ // when the batchsize is not a multiple of 16.
477
+ bool a_sh_wr_pred[a_sh_wr_iters];
478
+ #pragma unroll
479
+ for (int i = 0; i < a_sh_wr_iters; i++) {
480
+ int a_idx = a_sh_wr_delta * i + a_sh_wr;
481
+ int row = a_idx / a_gl_rd_delta_o;
482
+ if (row >= prob_m) {
483
+ a_sh_wr_pred[i] = false;
484
+ } else {
485
+ a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
486
+ }
487
+ }
488
+
489
+ // To ensure that writing and reading A tiles to/from shared memory, the
490
+ // latter in fragment format, is fully bank conflict free, we need to use a
491
+ // rather fancy XOR-based layout. The key here is that neither reads nor
492
+ // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
493
+ // same shared memory banks. Further, it seems (based on NSight-Compute) that
494
+ // each warp must also write a consecutive memory segment?
495
+ auto transform_a = [&](int i) {
496
+ int row = i / a_gl_rd_delta_o;
497
+ return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
498
+ };
499
+ // Since the computation of this remapping is non-trivial and, due to our main
500
+ // loop unrolls, all shared memory accesses are static, we simply precompute
501
+ // both transformed reads and writes.
502
+ int a_sh_wr_trans[a_sh_wr_iters];
503
+ #pragma unroll
504
+ for (int i = 0; i < a_sh_wr_iters; i++)
505
+ a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
506
+ int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
507
+ #pragma unroll
508
+ for (int i = 0; i < b_sh_wr_iters; i++) {
509
+ #pragma unroll
510
+ for (int j = 0; j < thread_m_blocks; j++)
511
+ a_sh_rd_trans[i][j] =
512
+ transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
513
+ }
514
+
515
+ // Since B-accesses have non-constant stride they have to be computed at
516
+ // runtime; we break dependencies between subsequent accesses with a tile by
517
+ // maintining multiple pointers (we have enough registers), a tiny
518
+ // optimization.
519
+ const int4* B_ptr[b_sh_wr_iters];
520
+ #pragma unroll
521
+ for (int i = 0; i < b_sh_wr_iters; i++)
522
+ B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
523
+
524
+ // Register storage for double buffer of shared memory reads.
525
+ FragA frag_a[2][thread_m_blocks];
526
+ I4 frag_b_quant[2][b_thread_vecs];
527
+ FragC frag_c[thread_m_blocks][4][2];
528
+ FragS frag_s[2][4]; // No act-order
529
+ FragS act_frag_s[2][4][4]; // For act-order
530
+
531
+ // Zero accumulators.
532
+ auto zero_accums = [&]() {
533
+ #pragma unroll
534
+ for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
535
+ reinterpret_cast<float*>(frag_c)[i] = 0;
536
+ };
537
+
538
+ auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
539
+ int last_group_id) {
540
+ sh_first_group_id = first_group_id;
541
+ sh_num_groups = last_group_id - first_group_id + 1;
542
+
543
+ if (sh_num_groups < sh_max_num_groups) {
544
+ sh_num_groups = sh_max_num_groups;
545
+ }
546
+
547
+ if (sh_first_group_id + sh_num_groups > num_groups) {
548
+ sh_num_groups = num_groups - sh_first_group_id;
549
+ }
550
+
551
+ int row_offset = first_group_id * s_gl_stride;
552
+
553
+ if (is_async) {
554
+ for (int i = 0; i < sh_num_groups; i++) {
555
+ if (threadIdx.x < s_sh_stride) {
556
+ cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
557
+ &scales_ptr[row_offset + (i * s_gl_stride) +
558
+ slice_n_offset + threadIdx.x]);
559
+ }
560
+ }
561
+ } else {
562
+ for (int i = 0; i < sh_num_groups; i++) {
563
+ if (threadIdx.x < s_sh_stride) {
564
+ sh_s[(i * s_sh_stride) + threadIdx.x] =
565
+ scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
566
+ threadIdx.x];
567
+ }
568
+ }
569
+ }
570
+ };
571
+ // Asynchronously fetch the next A, B and s tile from global to the next
572
+ // shared memory pipeline location.
573
+ auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
574
+ if (pred) {
575
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
576
+ #pragma unroll
577
+ for (int i = 0; i < a_sh_wr_iters; i++) {
578
+ int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off;
579
+ int row = a_idx / a_gl_stride;
580
+ int sorted_row =
581
+ replicate_input ? sorted_ids[row] / topk : sorted_ids[row];
582
+ int new_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride;
583
+ if (sorted_row < tot_m * (replicate_input ? 1 : topk) &&
584
+ new_idx < a_gl_stride * tot_m * (replicate_input ? 1 : topk)) {
585
+ cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[new_idx],
586
+ a_sh_wr_pred[i]);
587
+ }
588
+ }
589
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
590
+ #pragma unroll
591
+ for (int i = 0; i < b_sh_wr_iters; i++) {
592
+ #pragma unroll
593
+ for (int j = 0; j < b_thread_vecs; j++) {
594
+ cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
595
+ }
596
+ B_ptr[i] += b_gl_rd_delta_o;
597
+ }
598
+
599
+ if constexpr (has_act_order) {
600
+ // Fetch g_idx thread-block portion
601
+ int full_pipe = a_off;
602
+ int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
603
+ if (cur_k < prob_k && cur_k < slice_k_finish) {
604
+ int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
605
+
606
+ int4 const* cur_g_idx_stage_ptr =
607
+ reinterpret_cast<int4 const*>(&g_idx[cur_k]);
608
+
609
+ if (threadIdx.x < g_idx_stage) {
610
+ cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
611
+ &cur_g_idx_stage_ptr[threadIdx.x]);
612
+ }
613
+ }
614
+ } else {
615
+ if constexpr (group_blocks != -1) {
616
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
617
+
618
+ if constexpr (group_blocks >= thread_k_blocks) {
619
+ // Only fetch scales if this tile starts a new group
620
+ if (pipe % (group_blocks / thread_k_blocks) == 0) {
621
+ if (s_sh_wr_pred) {
622
+ cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
623
+ }
624
+ s_gl_rd += s_gl_rd_delta;
625
+ }
626
+ } else {
627
+ for (int i = 0; i < s_tb_groups; i++) {
628
+ if (s_sh_wr_pred) {
629
+ cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
630
+ &scales_ptr[s_gl_rd]);
631
+ }
632
+ s_gl_rd += s_gl_rd_delta;
633
+ }
634
+ }
635
+ }
636
+ }
637
+ }
638
+ // Insert a fence even when we are winding down the pipeline to ensure that
639
+ // waiting is also correct at this point.
640
+ cp_async_fence();
641
+ };
642
+
643
+ // TODO we are currently hitting illegal memory accesses when fetching
644
+ // sorted_ids to shared data: fix this
645
+ auto fetch_sorted_ids_to_shared = [&]() {
646
+ const int mpt = ceildiv(prob_m, threads);
647
+ for (int i = 0; i < mpt; i++) {
648
+ if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
649
+ sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
650
+ sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
651
+ }
652
+ }
653
+ };
654
+
655
+ // Wait until the next thread tile has been loaded to shared memory.
656
+ auto wait_for_stage = [&]() {
657
+ // We only have `stages - 2` active fetches since we are double buffering
658
+ // and can only issue the next fetch when it is guaranteed that the previous
659
+ // shared memory load is fully complete (as it may otherwise be
660
+ // overwritten).
661
+ cp_async_wait<stages - 2>();
662
+ __syncthreads();
663
+ };
664
+
665
+ // Load the next sub-tile from the current location in the shared memory pipe
666
+ // into the current register buffer.
667
+ auto fetch_to_registers = [&](int k, int pipe) {
668
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
669
+ #pragma unroll
670
+ for (int i = 0; i < thread_m_blocks; i++)
671
+ ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
672
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
673
+
674
+ #pragma unroll
675
+ for (int i = 0; i < b_thread_vecs; i++) {
676
+ frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
677
+ &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
678
+ }
679
+ };
680
+
681
+ bool is_same_group[stages];
682
+ int same_group_id[stages];
683
+
684
+ auto init_same_group = [&](int pipe) {
685
+ if constexpr (!has_act_order) {
686
+ is_same_group[pipe] = false;
687
+ same_group_id[pipe] = 0;
688
+ return;
689
+ }
690
+
691
+ int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
692
+ int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
693
+
694
+ int group_id_1 = sh_g_idx_int_ptr[0];
695
+ int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
696
+
697
+ is_same_group[pipe] = group_id_1 == group_id_2;
698
+ same_group_id[pipe] = group_id_1;
699
+ };
700
+
701
+ auto fetch_scales_to_registers = [&](int k, int full_pipe) {
702
+ int pipe = full_pipe % stages;
703
+
704
+ if constexpr (!has_act_order) {
705
+ // No act-order case
706
+ if constexpr (group_blocks != -1) {
707
+ if constexpr (group_blocks >= thread_k_blocks) {
708
+ int4* sh_s_stage =
709
+ sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
710
+ (pipe / (group_blocks / thread_k_blocks)));
711
+ reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
712
+ } else {
713
+ int warp_id = threadIdx.x / 32;
714
+ int n_warps = thread_n_blocks / 4;
715
+
716
+ int warp_row = warp_id / n_warps;
717
+
718
+ int cur_k = warp_row * 16;
719
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
720
+
721
+ int k_blocks = cur_k / 16;
722
+ int cur_group_id = k_blocks / group_blocks;
723
+
724
+ int4* sh_s_stage = sh_s + s_sh_stage * pipe;
725
+
726
+ reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
727
+ sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
728
+ }
729
+ }
730
+
731
+ return;
732
+ }
733
+
734
+ // Act-order case
735
+
736
+ // Determine K of the "current" thread-block
737
+ int cur_k = slice_k_start + tb_k * full_pipe;
738
+ if (cur_k >= prob_k || cur_k >= slice_k_finish) {
739
+ return;
740
+ }
741
+
742
+ // Reset (to current thread-block) since we read g_idx portion from the
743
+ // shared memory
744
+ cur_k = 0;
745
+
746
+ // Progress to current iteration
747
+ cur_k += k_iter_size * (k % b_sh_wr_iters);
748
+
749
+ // Determine "position" inside the thread-block (based on warp and
750
+ // thread-id)
751
+ int warp_id = threadIdx.x / 32;
752
+ int n_warps =
753
+ thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
754
+
755
+ int warp_row = warp_id / n_warps;
756
+ int warp_col = warp_id % n_warps;
757
+
758
+ cur_k += warp_row * 16;
759
+
760
+ int th_id = threadIdx.x % 32;
761
+ cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
762
+
763
+ int s_col_shift =
764
+ /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
765
+ (th_id / 4) * act_s_col_stride;
766
+
767
+ if (is_same_group[pipe]) {
768
+ if (k % 2 == 0) {
769
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
770
+ sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
771
+ s_col_shift];
772
+ } else {
773
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
774
+ *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
775
+ }
776
+
777
+ for (int i = 1; i < 4; i++) {
778
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
779
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
780
+ }
781
+ return;
782
+ }
783
+
784
+ int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
785
+ int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
786
+
787
+ constexpr int k_frag_offsets[4] = {0, 1, 8,
788
+ 9}; // Tensor core offsets per thread
789
+
790
+ #pragma unroll
791
+ for (int i = 0; i < 4; i++) {
792
+ int actual_k = cur_k + k_frag_offsets[i];
793
+
794
+ int group_id = sh_g_idx_int_ptr[actual_k];
795
+ int rel_group_id = group_id - sh_first_group_id;
796
+
797
+ *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
798
+ sh_s[rel_group_id * s_sh_stride + s_col_shift];
799
+ }
800
+ };
801
+
802
+ // Execute the actual tensor core matmul of a sub-tile.
803
+ auto matmul = [&](int k) {
804
+ // We have the m dimension as the inner loop in order to encourage overlapping
805
+ // dequantization and matmul operations.
806
+ #pragma unroll
807
+ for (int j = 0; j < 4; j++) {
808
+ int b_quant_0, b_quant_1;
809
+ if constexpr (w_type.size_bits() == 4) {
810
+ b_quant_0 = frag_b_quant[k % 2][0][j];
811
+ b_quant_1 = b_quant_0 >> 8;
812
+ } else {
813
+ static_assert(w_type.size_bits() == 8);
814
+ int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
815
+ b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
816
+ b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
817
+ }
818
+
819
+ FragB frag_b0 = dequant<w_type_id>(b_quant_0);
820
+ FragB frag_b1 = dequant<w_type_id>(b_quant_1);
821
+
822
+ // Apply scale to frag_b0
823
+ if constexpr (has_act_order) {
824
+ scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
825
+ act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
826
+ } else {
827
+ if constexpr (group_blocks != -1) {
828
+ scale(frag_b0, frag_s[k % 2][j], 0);
829
+ }
830
+ }
831
+
832
+ // Apply scale to frag_b1
833
+ if constexpr (has_act_order) {
834
+ scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
835
+ act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
836
+
837
+ } else {
838
+ if constexpr (group_blocks != -1) {
839
+ scale(frag_b1, frag_s[k % 2][j], 1);
840
+ }
841
+ }
842
+
843
+ #pragma unroll
844
+ for (int i = 0; i < thread_m_blocks; i++) {
845
+ mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
846
+ mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
847
+ }
848
+ }
849
+ };
850
+
851
+ // Since we slice across the k dimension of a tile in order to increase the
852
+ // number of warps while keeping the n dimension of a tile reasonable, we have
853
+ // multiple warps that accumulate their partial sums of the same output
854
+ // location; which we have to reduce over in the end. We do in shared memory.
855
+ auto thread_block_reduce = [&]() {
856
+ constexpr int red_off = threads / b_sh_stride_threads / 2;
857
+ if (red_off >= 1) {
858
+ int red_idx = threadIdx.x / b_sh_stride_threads;
859
+ constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
860
+ constexpr int red_sh_delta = b_sh_stride_threads;
861
+ int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
862
+ (threadIdx.x % b_sh_stride_threads);
863
+
864
+ // Parallel logarithmic shared memory reduction. We make sure to avoid any
865
+ // unnecessary read or write iterations, e.g., for two warps we write only
866
+ // once by warp 1 and read only once by warp 0.
867
+
868
+ #pragma unroll
869
+ for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
870
+ #pragma unroll
871
+ for (int i = red_off; i > 0; i /= 2) {
872
+ if (i <= red_idx && red_idx < 2 * i) {
873
+ #pragma unroll
874
+ for (int j = 0; j < 4 * 2; j++) {
875
+ int red_sh_wr =
876
+ red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
877
+ if (i < red_off) {
878
+ float* c_rd =
879
+ reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
880
+ float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
881
+ #pragma unroll
882
+ for (int k = 0; k < 4; k++)
883
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
884
+ c_rd[k] + c_wr[k];
885
+ }
886
+ sh[red_sh_wr] =
887
+ reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
888
+ }
889
+ }
890
+ __syncthreads();
891
+ }
892
+ if (red_idx == 0) {
893
+ #pragma unroll
894
+ for (int i = 0; i < 4 * 2; i++) {
895
+ float* c_rd =
896
+ reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
897
+ #pragma unroll
898
+ for (int j = 0; j < 4; j++)
899
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
900
+ c_rd[j];
901
+ }
902
+ }
903
+ __syncthreads();
904
+ }
905
+ }
906
+ };
907
+
908
+ // Since multiple threadblocks may process parts of the same column slice, we
909
+ // finally have to globally reduce over the results. As the striped
910
+ // partitioning minimizes the number of such reductions and our outputs are
911
+ // usually rather small, we perform this reduction serially in L2 cache.
912
+ auto global_reduce = [&](bool first = false, bool last = false) {
913
+ // We are very careful here to reduce directly in the output buffer to
914
+ // maximize L2 cache utilization in this step. To do this, we write out
915
+ // results in FP16 (but still reduce with FP32 compute).
916
+ constexpr int active_threads = 32 * thread_n_blocks / 4;
917
+ if (threadIdx.x < active_threads) {
918
+ int c_gl_stride = prob_n / 8;
919
+ int c_gl_wr_delta_o = 8 * c_gl_stride;
920
+ int c_gl_wr_delta_i = 4 * (active_threads / 32);
921
+ int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
922
+ 4 * (threadIdx.x / 32) + threadIdx.x % 4;
923
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
924
+ constexpr int c_sh_wr_delta = active_threads;
925
+ int c_sh_wr = threadIdx.x;
926
+
927
+ int row = (threadIdx.x % 32) / 4;
928
+
929
+ if (!first) {
930
+ // Interestingly, doing direct global accesses here really seems to mess up
931
+ // the compiler and lead to slowdowns, hence we also use async-copies even
932
+ // though these fetches are not actually asynchronous.
933
+ #pragma unroll
934
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
935
+ int c_idx =
936
+ c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
937
+ int sorted_row = sorted_ids[c_idx / c_gl_stride];
938
+ int new_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride;
939
+ cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i], &C[new_idx],
940
+ sorted_row < tot_m * topk &&
941
+ (8 * (i / 2) + row < prob_m &&
942
+ (i < (thread_m_blocks - 1) * 4 ||
943
+ sorted_ids[8 * (i / 2) + row] < tot_m * topk)));
944
+ }
945
+ cp_async_fence();
946
+ cp_async_wait<0>();
947
+ }
948
+
949
+ #pragma unroll
950
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
951
+ if (8 * (i / 2) + row < prob_m &&
952
+ (i < (thread_m_blocks - 1) * 4 ||
953
+ sorted_ids[8 * (i / 2) + row] < tot_m * topk)) {
954
+ if (!first) {
955
+ int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
956
+ #pragma unroll
957
+ for (int j = 0; j < 2 * 4; j++) {
958
+ reinterpret_cast<float*>(
959
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
960
+ __half2float(reinterpret_cast<__half*>(&c_red)[j]);
961
+ }
962
+ }
963
+ if (!last) {
964
+ int4 c;
965
+ #pragma unroll
966
+ for (int j = 0; j < 2 * 4; j++) {
967
+ reinterpret_cast<__half*>(&c)[j] =
968
+ __float2half(reinterpret_cast<float*>(
969
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
970
+ }
971
+ int c_idx =
972
+ c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
973
+ int row = sorted_ids[c_idx / c_gl_stride];
974
+ if (row < tot_m * topk) {
975
+ int new_idx = row * c_gl_stride + c_idx % c_gl_stride;
976
+ C[new_idx] = c;
977
+ }
978
+ }
979
+ }
980
+ }
981
+ }
982
+ };
983
+
984
+ // Write out the reduce final result in the correct layout. We only actually
985
+ // reshuffle matrix fragments in this step, the reduction above is performed
986
+ // in fragment layout.
987
+ auto write_result = [&]() {
988
+ int c_gl_stride = prob_n / 8;
989
+ constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
990
+ int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
991
+ constexpr int c_sh_rd_delta =
992
+ c_sh_stride * (threads / (2 * thread_n_blocks));
993
+
994
+ int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
995
+ (threadIdx.x % (2 * thread_n_blocks));
996
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
997
+ int c_sh_wr =
998
+ (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
999
+ c_sh_wr += 32 * (threadIdx.x / 32);
1000
+ int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1001
+ (threadIdx.x % (2 * thread_n_blocks));
1002
+
1003
+ int c_gl_wr_end = c_gl_stride * prob_m;
1004
+
1005
+ // We first reorder in shared memory to guarantee the most efficient final
1006
+ // global write patterns
1007
+ auto write = [&](int idx, float c0, float c1, FragS& s) {
1008
+ half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1009
+
1010
+ // For per-column quantization we finally apply the scale here (only for
1011
+ // 4-bit)
1012
+ if constexpr (!has_act_order && group_blocks == -1 &&
1013
+ w_type.size_bits() == 4) {
1014
+ res = __hmul2(res, s[0]);
1015
+ }
1016
+
1017
+ ((half2*)sh)[idx] = res;
1018
+ };
1019
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1020
+ #pragma unroll
1021
+ for (int i = 0; i < thread_m_blocks; i++) {
1022
+ #pragma unroll
1023
+ for (int j = 0; j < 4; j++) {
1024
+ int wr = c_sh_wr + 8 * j;
1025
+ write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
1026
+ frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
1027
+ write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
1028
+ frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
1029
+ write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
1030
+ frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
1031
+ write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
1032
+ frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
1033
+ }
1034
+ c_sh_wr += 16 * (4 * c_sh_stride);
1035
+ }
1036
+ }
1037
+ __syncthreads();
1038
+
1039
+ #pragma unroll
1040
+ for (int i = 0;
1041
+ i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1042
+ i++) {
1043
+ if (c_gl_wr < c_gl_wr_end) {
1044
+ int row = sorted_ids[c_gl_wr / c_gl_stride];
1045
+ if (row < tot_m * topk) {
1046
+ int off = row * c_gl_stride + c_gl_wr % c_gl_stride;
1047
+ if (!apply_weights) {
1048
+ C[off] = sh[c_sh_rd];
1049
+ } else {
1050
+ __half* ctrg = reinterpret_cast<__half*>(&C[off]);
1051
+ __half* csrc = reinterpret_cast<__half*>(&sh[c_sh_rd]);
1052
+ for (int j = 0; j < 8; ++j) {
1053
+ ctrg[j] = __float2half(topk_weights[row] * __half2float(csrc[j]));
1054
+ }
1055
+ }
1056
+ c_gl_wr += c_gl_wr_delta;
1057
+ c_sh_rd += c_sh_rd_delta;
1058
+ }
1059
+ }
1060
+ }
1061
+ };
1062
+
1063
+ // Start global fetch and register load pipelines.
1064
+ auto start_pipes = [&]() {
1065
+ // TODO re-enable after fixing this function
1066
+ // fetch_sorted_ids_to_shared();
1067
+ // __syncthreads();
1068
+
1069
+ #pragma unroll
1070
+ for (int i = 0; i < stages - 1; i++) {
1071
+ if (has_act_order && i == 0) {
1072
+ int last_g_idx = slice_k_start + stages * tb_k * 2;
1073
+ if (last_g_idx >= prob_k) {
1074
+ last_g_idx = prob_k - 1;
1075
+ }
1076
+ fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1077
+ }
1078
+ fetch_to_shared(i, i, i < slice_iters);
1079
+ }
1080
+
1081
+ zero_accums();
1082
+ wait_for_stage();
1083
+ init_same_group(0);
1084
+ fetch_to_registers(0, 0);
1085
+ fetch_scales_to_registers(0, 0);
1086
+ a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1087
+ slice_k_start_shared_fetch += tb_k * (stages - 1);
1088
+ };
1089
+ if (slice_iters) {
1090
+ start_pipes();
1091
+ }
1092
+
1093
+ // Main loop.
1094
+ while (slice_iters) {
1095
+ // We unroll over both the global fetch and the register load pipeline to
1096
+ // ensure all shared memory accesses are static. Note that both pipelines
1097
+ // have even length meaning that the next iteration will always start at
1098
+ // index 0.
1099
+ #pragma unroll
1100
+ for (int pipe = 0; pipe < stages;) {
1101
+ #pragma unroll
1102
+ for (int k = 0; k < b_sh_wr_iters; k++) {
1103
+ fetch_to_registers(k + 1, pipe % stages);
1104
+ fetch_scales_to_registers(k + 1, pipe);
1105
+ if (k == b_sh_wr_iters - 2) {
1106
+ fetch_to_shared((pipe + stages - 1) % stages, pipe,
1107
+ slice_iters >= stages);
1108
+ pipe++;
1109
+ wait_for_stage();
1110
+ init_same_group(pipe % stages);
1111
+ }
1112
+ matmul(k);
1113
+ }
1114
+ slice_iters--;
1115
+ if (slice_iters == 0) {
1116
+ break;
1117
+ }
1118
+ }
1119
+
1120
+ a_gl_rd += a_gl_rd_delta_o * stages;
1121
+ slice_k_start += tb_k * stages;
1122
+ slice_k_start_shared_fetch += tb_k * stages;
1123
+
1124
+ if constexpr (has_act_order) {
1125
+ int first_group_id = g_idx[slice_k_start];
1126
+ int last_g_idx = slice_k_start + stages * tb_k * 2;
1127
+ if (last_g_idx >= prob_k) {
1128
+ last_g_idx = prob_k - 1;
1129
+ }
1130
+ int last_group_id = g_idx[last_g_idx];
1131
+ if (last_group_id >= sh_first_group_id + sh_num_groups) {
1132
+ fetch_scales_to_shared(false, first_group_id, last_group_id);
1133
+ __syncthreads();
1134
+ }
1135
+ }
1136
+
1137
+ // Process results and, if necessary, proceed to the next column slice.
1138
+ // While this pattern may not be the most readable, other ways of writing
1139
+ // the loop seemed to noticeably worse performance after compilation.
1140
+ if (slice_iters == 0) {
1141
+ cp_async_wait<0>();
1142
+ bool last = slice_idx == slice_count - 1;
1143
+ if constexpr (!has_act_order && group_blocks == -1) {
1144
+ if constexpr (w_type.size_bits() == 8) {
1145
+ if (s_sh_wr_pred) {
1146
+ cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1147
+ }
1148
+ cp_async_fence();
1149
+ } else {
1150
+ // For 4-bit per-column scales, we only fetch them here in the
1151
+ // final step before write-out
1152
+ if (last) {
1153
+ if (s_sh_wr_pred) {
1154
+ cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1155
+ }
1156
+ cp_async_fence();
1157
+ }
1158
+ }
1159
+ }
1160
+
1161
+ thread_block_reduce();
1162
+ if constexpr (!has_act_order && group_blocks == -1) {
1163
+ if constexpr (w_type.size_bits() == 8) {
1164
+ cp_async_wait<0>();
1165
+ __syncthreads();
1166
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1167
+ reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1168
+ reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1169
+ }
1170
+
1171
+ } else {
1172
+ if (last) {
1173
+ cp_async_wait<0>();
1174
+ __syncthreads();
1175
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1176
+ reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1177
+ reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1178
+ }
1179
+ }
1180
+ }
1181
+ }
1182
+
1183
+ // For 8-bit channelwise, we apply the scale before the global reduction
1184
+ // that converts the fp32 results to fp16 (so that we avoid possible
1185
+ // overflow in fp16)
1186
+ if constexpr (!has_act_order && group_blocks == -1 &&
1187
+ w_type.size_bits() == 8) {
1188
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
1189
+ #pragma unroll
1190
+ for (int i = 0; i < thread_m_blocks; i++) {
1191
+ #pragma unroll
1192
+ for (int j = 0; j < 4; j++) {
1193
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1194
+ frag_s[j / 2][2 * (j % 2) + 0]);
1195
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1196
+ frag_s[j / 2][2 * (j % 2) + 0]);
1197
+
1198
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1199
+ frag_s[j / 2][2 * (j % 2) + 1]);
1200
+ scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1201
+ frag_s[j / 2][2 * (j % 2) + 1]);
1202
+ }
1203
+ }
1204
+ }
1205
+ }
1206
+
1207
+ if (slice_count > 1) { // only globally reduce if there is more than one
1208
+ // block in a slice
1209
+ barrier_acquire(&locks[slice_col], slice_idx);
1210
+ global_reduce(slice_idx == 0, last);
1211
+ barrier_release(&locks[slice_col], last);
1212
+ }
1213
+ if (last) // only the last block in a slice actually writes the result
1214
+ write_result();
1215
+ slice_row = 0;
1216
+ slice_col_par++;
1217
+ slice_col++;
1218
+ init_slice();
1219
+ if (slice_iters) {
1220
+ a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1221
+ (threadIdx.x % a_gl_rd_delta_o);
1222
+ #pragma unroll
1223
+ for (int i = 0; i < b_sh_wr_iters; i++)
1224
+ B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1225
+ if (slice_col == 0) {
1226
+ #pragma unroll
1227
+ for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1228
+ }
1229
+
1230
+ // Update slice k/n for scales loading
1231
+ if constexpr (has_act_order) {
1232
+ slice_k_start = tb_k * slice_row;
1233
+ slice_k_finish = slice_k_start + tb_k * slice_iters;
1234
+ slice_k_start_shared_fetch = slice_k_start;
1235
+ slice_n_offset = act_s_col_tb_stride * slice_col;
1236
+
1237
+ } else {
1238
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1239
+ }
1240
+ start_pipes();
1241
+ }
1242
+ }
1243
+ }
1244
+}
1245
+
1246
+template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1247
+ const int threads, // number of threads in a threadblock
1248
+ const int thread_n_blocks, // same for n dimension (output)
1249
+ const int thread_k_blocks, // same for k dimension (reduction)
1250
+ const int stages, // number of stages for the async global->shared
1251
+ // fetch pipeline
1252
+ const bool has_act_order, // whether act_order is enabled
1253
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
1254
+ // with a separate quantization scale
1255
+ >
1256
+__global__ void MarlinMoE(
1257
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
1258
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1259
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
1260
+ const int* __restrict__ sorted_ids_base, // int32 sorted ids of experts
1261
+ const float* __restrict__ topk_weights, // float topk weights
1262
+ const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1263
+ // (k/groupsize)xn
1264
+ const int* __restrict__ g_idx, // int32 group indices of shape k
1265
+ const int* __restrict__ expert_offsets,
1266
+ int num_groups, // number of scale groups per output channel
1267
+ int expert_idx, // idx of current expert
1268
+ int num_experts, // number of experts
1269
+ int topk, // topk parameter of moe
1270
+ int prob_m, // batch dimension m
1271
+ int prob_n, // output dimension n
1272
+ int prob_k, // reduction dimension k
1273
+ int tot_m, // total number of rows in A and C
1274
+ int* locks, // extra global storage for barrier synchronization
1275
+ bool replicate_input, // do we use the same input for each expert?
1276
+ bool apply_weights, // apply weights to output
1277
+ int current_m_block, // current m block to start kernel computation from
1278
+ int max_par, // maximum parallelism
1279
+ int cfg_max_m_blocks // upper bound on m blocks
1280
+) {
1281
+ int m_block_ctr = current_m_block;
1282
+
1283
+ const int* sorted_ids_expert =
1284
+ sorted_ids_base + expert_offsets[expert_idx] + m_block_ctr * 4 * max_par;
1285
+ int tot_its = expert_offsets[expert_idx + 1] - expert_offsets[expert_idx];
1286
+ if (tot_its == 0) {
1287
+ return;
1288
+ }
1289
+ int tot_m_blocks = ceildiv(tot_its, 16);
1290
+ int pad = 16 * tot_m_blocks - tot_its;
1291
+
1292
+ if (m_block_ctr >= tot_m_blocks) {
1293
+ return;
1294
+ }
1295
+
1296
+ int max_block = tot_m_blocks - m_block_ctr;
1297
+ prob_m = tot_its - 16 * m_block_ctr;
1298
+
1299
+ int par = 1;
1300
+ if (max_block > cfg_max_m_blocks) {
1301
+ // Note that parallel > 1 currently only works for inputs without any
1302
+ // padding
1303
+ par = (16 * max_block - pad) / (16 * cfg_max_m_blocks);
1304
+ if (par > max_par) par = max_par;
1305
+ prob_m = (16 * cfg_max_m_blocks) * par;
1306
+ m_block_ctr += cfg_max_m_blocks * (par - 1);
1307
+ max_block = cfg_max_m_blocks;
1308
+ }
1309
+
1310
+ if (max_block == 1) {
1311
+ MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1312
+ stages, has_act_order, group_blocks>(
1313
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1314
+ expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1315
+ prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1316
+ current_m_block);
1317
+ } else if (max_block == 2) {
1318
+ MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1319
+ stages, has_act_order, group_blocks>(
1320
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1321
+ expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1322
+ prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1323
+ current_m_block);
1324
+ } else if (max_block == 3) {
1325
+ MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1326
+ stages, has_act_order, group_blocks>(
1327
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1328
+ expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1329
+ prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1330
+ current_m_block);
1331
+ } else {
1332
+ MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1333
+ stages, has_act_order, group_blocks>(
1334
+ A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1335
+ expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1336
+ prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1337
+ current_m_block);
1338
+ }
1339
+}
1340
+
1341
+#else
1342
+
1343
+template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1344
+ const int threads, // number of threads in a threadblock
1345
+ const int thread_n_blocks, // same for n dimension (output)
1346
+ const int thread_k_blocks, // same for k dimension (reduction)
1347
+ const int stages, // number of stages for the async global->shared
1348
+ // fetch pipeline
1349
+ const bool has_act_order, // whether act_order is enabled
1350
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
1351
+ // with a separate quantization scale
1352
+ >
1353
+__global__ void MarlinMoE(
1354
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
1355
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1356
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
1357
+ const int* __restrict__ sorted_ids, // int32 sorted ids of experts
1358
+ const float* __restrict__ topk_weights, // float topk weights
1359
+ const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1360
+ // (k/groupsize)xn
1361
+ const int* __restrict__ g_idx, // int32 group indices of shape k
1362
+ const int* __restrict__ expert_offsets,
1363
+ int num_groups, // number of scale groups per output channel
1364
+ int expert_idx, // idx of current expert
1365
+ int num_experts, // number of experts
1366
+ int topk, // topk parameter of moe
1367
+ int prob_m, // batch dimension m
1368
+ int prob_n, // output dimension n
1369
+ int prob_k, // reduction dimension k
1370
+ int tot_m, // total number of rows in A and C
1371
+ int* locks, // extra global storage for barrier synchronization
1372
+ bool replicate_input, // do we use the same input for each expert?
1373
+ bool apply_weights, // apply weights to output
1374
+ int current_m_block, // current m block to start kernel computation from
1375
+ int max_par, // maximum parallelism
1376
+ int cfg_max_m_blocks // upper bound on m blocks
1377
+
1378
+) {
1379
+ // Marlin is not implemented yet for SM < 8.0
1380
+ assert(false);
1381
+ return;
1382
+}
1383
+
1384
+#endif
1385
+
1386
+// 8 warps are a good choice since every SM has 4 schedulers and having more
1387
+// than 1 warp per schedule allows some more latency hiding. At the same time,
1388
+// we want relatively few warps to have many registers per warp and small tiles.
1389
+const int USER_THREADS =
1390
+ 256; // Note: This is only used with user-provided thread_k/n
1391
+const int STAGES = 4; // 4 pipeline stages fit into shared memory
1392
+// const int SHARED_MEM =
1393
+// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1394
+
1395
+static constexpr int min_thread_n = 64;
1396
+static constexpr int min_thread_k = 64;
1397
+
1398
+#define __CALL_IF_MOE(W_TYPE, THREAD_N_BLOCKS, THREAD_K_BLOCKS, HAS_ACT_ORDER, \
1399
+ GROUP_BLOCKS, NUM_THREADS) \
1400
+ else if (q_type == W_TYPE && thread_n_blocks == THREAD_N_BLOCKS && \
1401
+ thread_k_blocks == THREAD_K_BLOCKS && \
1402
+ has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1403
+ num_threads == NUM_THREADS) { \
1404
+ cudaFuncSetAttribute( \
1405
+ MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1406
+ STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1407
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1408
+ MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1409
+ STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1410
+ <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1411
+ A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1412
+ g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1413
+ num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1414
+ replicate_input, apply_weights, m_block, max_par, \
1415
+ cfg_max_m_blocks); \
1416
+ }
1417
+
1418
+#define GPTQ_CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1419
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1420
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1421
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1422
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1423
+ __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1424
+
1425
+} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu
new
+29
@@ -0,0 +1,29 @@
1
+#include "marlin_moe_kernel_ku4b8.h"
2
+
3
+namespace marlin_moe {
4
+
5
+// We return bool so we can create these different kernel calls as a sequence
6
+// of if-elseif's.
7
+bool call_marlin_moe_kernel_ku4b8(
8
+ vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9
+ bool has_act_order, int group_blocks, int num_threads, int blocks,
10
+ int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11
+ const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12
+ const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13
+ int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14
+ int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15
+ bool replicate_input, bool apply_weights, int m_block, int max_par,
16
+ int cfg_max_m_blocks) {
17
+ if (false) {
18
+ }
19
+ GPTQ_CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
20
+ GPTQ_CALL_IF_MOE(vllm::kU4B8, 8, 8, 256)
21
+ GPTQ_CALL_IF_MOE(vllm::kU4B8, 8, 4, 128)
22
+ GPTQ_CALL_IF_MOE(vllm::kU4B8, 4, 8, 128)
23
+ else {
24
+ return false;
25
+ }
26
+ return true;
27
+}
28
+
29
+} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h
new
+20
@@ -0,0 +1,20 @@
1
+#pragma once
2
+
3
+#include "marlin_moe_kernel.h"
4
+
5
+namespace marlin_moe {
6
+
7
+// We return bool so we can create these different kernel calls as a sequence
8
+// of if-elseif's.
9
+bool call_marlin_moe_kernel_ku4b8(
10
+ vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
11
+ bool has_act_order, int group_blocks, int num_threads, int blocks,
12
+ int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
13
+ const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
14
+ const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
15
+ int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
16
+ int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
17
+ bool replicate_input, bool apply_weights, int m_block, int max_par,
18
+ int cfg_max_m_blocks);
19
+
20
+} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu
new
+29
@@ -0,0 +1,29 @@
1
+#include "marlin_moe_kernel_ku8b128.h"
2
+
3
+namespace marlin_moe {
4
+
5
+// We return bool so we can create these different kernel calls as a sequence
6
+// of if-elseif's.
7
+bool call_marlin_moe_kernel_ku8b128(
8
+ vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9
+ bool has_act_order, int group_blocks, int num_threads, int blocks,
10
+ int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11
+ const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12
+ const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13
+ int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14
+ int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15
+ bool replicate_input, bool apply_weights, int m_block, int max_par,
16
+ int cfg_max_m_blocks) {
17
+ if (false) {
18
+ }
19
+ GPTQ_CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
20
+ GPTQ_CALL_IF_MOE(vllm::kU8B128, 8, 8, 256)
21
+ GPTQ_CALL_IF_MOE(vllm::kU8B128, 8, 4, 128)
22
+ GPTQ_CALL_IF_MOE(vllm::kU8B128, 4, 8, 128)
23
+ else {
24
+ return false;
25
+ }
26
+ return true;
27
+}
28
+
29
+} // namespace marlin_moe
csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h
new
+18
@@ -0,0 +1,18 @@
1
+#pragma once
2
+
3
+#include "marlin_moe_kernel.h"
4
+
5
+namespace marlin_moe {
6
+
7
+bool call_marlin_moe_kernel_ku8b128(
8
+ vllm::ScalarType const& q_type, int thread_n_blocks, int thread_k_blocks,
9
+ bool has_act_order, int group_blocks, int num_threads, int blocks,
10
+ int max_shared_mem, cudaStream_t stream, const int4* A_ptr,
11
+ const int4* B_ptr, int4* C_ptr, const int* sorted_ids_ptr,
12
+ const float* topk_weights_ptr, const int4* s_ptr, const int* g_idx_ptr,
13
+ int* expert_offsets_ptr, int num_groups, int expert_idx, int num_experts,
14
+ int topk, int prob_m, int prob_n, int prob_k, int tot_m, int* locks,
15
+ bool replicate_input, bool apply_weights, int m_block, int max_par,
16
+ int cfg_max_m_blocks);
17
+
18
+}
csrc/moe/marlin_moe_ops.cu
+26
-1427
@@ -26,6 +26,8 @@
26
#include <iostream>
27
28
#include "core/scalar_type.hpp"
29
+#include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
30
+#include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
31
32
template <typename T>
33
inline std::string str(T x) {
@@ -34,230 +36,8 @@ inline std::string str(T x) {
36
37
namespace marlin_moe {
38
37
-constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
38
-
39
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
40
41
-// Instances of `Vec` are used to organize groups of >>registers<<, as needed
42
-// for instance as inputs to tensor core operations. Consequently, all
43
-// corresponding index accesses must be compile-time constants, which is why we
44
-// extensively use `#pragma unroll` throughout the kernel code to guarantee
45
-// this.
46
-template <typename T, int n>
47
-struct Vec {
48
- T elems[n];
49
- __device__ T& operator[](int i) { return elems[i]; }
50
-};
51
-
52
-using I4 = Vec<int, 4>;
53
-
54
-// Matrix fragments for tensor core instructions; their precise layout is
55
-// documented here:
56
-// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
57
-using FragA = Vec<half2, 4>;
58
-using FragB = Vec<half2, 2>;
59
-using FragC = Vec<float, 4>;
60
-using FragS = Vec<half2, 1>; // quantization scales
61
-
62
-// Predicated asynchronous global->shared copy; used for inputs A where we apply
63
-// predication to handle batchsizes that are not multiples of 16.
64
-__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
65
- bool pred = true) {
66
- const int BYTES = 16;
67
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
68
- asm volatile(
69
- "{\n"
70
- " .reg .pred p;\n"
71
- " setp.ne.b32 p, %0, 0;\n"
72
- " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
73
- "}\n" ::"r"((int)pred),
74
- "r"(smem), "l"(glob_ptr), "n"(BYTES));
75
-}
76
-
77
-// Asynchronous global->shared copy
78
-__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
79
- const int BYTES = 16;
80
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
81
- asm volatile(
82
- "{\n"
83
- " cp.async.cg.shared.global [%0], [%1], %2;\n"
84
- "}\n" ::"r"(smem),
85
- "l"(glob_ptr), "n"(BYTES));
86
-}
87
-
88
-// Async copy fence.
89
-__device__ inline void cp_async_fence() {
90
- asm volatile("cp.async.commit_group;\n" ::);
91
-}
92
-
93
-// Wait until at most `n` async copy stages are still pending.
94
-template <int n>
95
-__device__ inline void cp_async_wait() {
96
- asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
97
-}
98
-
99
-// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
100
-// output/accumulation.
101
-__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
102
- FragC& frag_c) {
103
- const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
104
- const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
105
- float* c = reinterpret_cast<float*>(&frag_c);
106
- asm volatile(
107
- "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
108
- "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
109
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
110
- : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
111
- "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
112
-}
113
-
114
-// Instruction for loading a full 16x16 matrix fragment of operand A from shared
115
-// memory, directly in tensor core layout.
116
-__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
117
- uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
118
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
119
- asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
120
- : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
121
- : "r"(smem));
122
-}
123
-
124
-// Lookup-table based 3-input logical operation; explicitly used for
125
-// dequantization as the compiler does not seem to automatically recognize it in
126
-// all cases.
127
-template <int lut>
128
-__device__ inline int lop3(int a, int b, int c) {
129
- int res;
130
- asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
131
- : "=r"(res)
132
- : "r"(a), "r"(b), "r"(c), "n"(lut));
133
- return res;
134
-}
135
-
136
-// Constructs destination register by taking bytes from 2 sources (based on
137
-// mask)
138
-template <int start_byte, int mask>
139
-__device__ inline uint32_t prmt(uint32_t a) {
140
- uint32_t res;
141
- asm volatile("prmt.b32 %0, %1, %2, %3;\n"
142
- : "=r"(res)
143
- : "r"(a), "n"(start_byte), "n"(mask));
144
- return res;
145
-}
146
-
147
-template <vllm::ScalarTypeId w_type_id>
148
-__device__ inline FragB dequant(int q);
149
-
150
-// Efficiently dequantize 4bit values packed in an int32 value into a full
151
-// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below,
152
-// with some small changes:
153
-// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287
154
-template <>
155
-__device__ inline FragB dequant<vllm::kU4B8.id()>(int q) {
156
- const int LO = 0x000f000f;
157
- const int HI = 0x00f000f0;
158
- const int EX = 0x64006400;
159
- // Guarantee that the `(a & b) | c` operations are LOP3s.
160
- int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
161
- int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
162
- // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
163
- // directly into `SUB` and `ADD`.
164
- const int SUB = 0x64086408;
165
- const int MUL = 0x2c002c00;
166
- const int ADD = 0xd480d480;
167
- FragB frag_b;
168
- frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
169
- *reinterpret_cast<const half2*>(&SUB));
170
- frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
171
- *reinterpret_cast<const half2*>(&MUL),
172
- *reinterpret_cast<const half2*>(&ADD));
173
- return frag_b;
174
-}
175
-
176
-// Fast Int8ToFp16: Efficiently dequantize 8bit int values to fp16
177
-// Reference:
178
-// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
179
-template <>
180
-__device__ inline FragB dequant<vllm::kU8B128.id()>(int q) {
181
- static constexpr uint32_t mask_for_elt_01 = 0x5250;
182
- static constexpr uint32_t mask_for_elt_23 = 0x5351;
183
- static constexpr uint32_t start_byte_for_fp16 = 0x64646464;
184
-
185
- uint32_t lo = prmt<start_byte_for_fp16, mask_for_elt_01>(q);
186
- uint32_t hi = prmt<start_byte_for_fp16, mask_for_elt_23>(q);
187
-
188
- static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480;
189
-
190
- FragB frag_b;
191
- frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
192
- *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
193
- frag_b[1] = __hsub2(*reinterpret_cast<half2*>(&hi),
194
- *reinterpret_cast<const half2*>(&I8s_TO_F16s_MAGIC_NUM));
195
- return frag_b;
196
-}
197
-
198
-// Multiply dequantized values by the corresponding quantization scale; used
199
-// only for grouped quantization.
200
-__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
201
- half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
202
- frag_b[0] = __hmul2(frag_b[0], s);
203
- frag_b[1] = __hmul2(frag_b[1], s);
204
-}
205
-
206
-// Given 2 floats multiply by 2 scales (halves)
207
-__device__ inline void scale_float(float* c, FragS& s) {
208
- __half* s_ptr = reinterpret_cast<__half*>(&s);
209
- c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
210
- c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
211
-}
212
-
213
-// Same as above, but for act_order (each K is multiplied individually)
214
-__device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
215
- FragS& frag_s_3, FragS& frag_s_4, int i) {
216
- __half2 s_val_1_2;
217
- s_val_1_2.x = reinterpret_cast<__half*>(&frag_s_1)[i];
218
- s_val_1_2.y = reinterpret_cast<__half*>(&frag_s_2)[i];
219
-
220
- __half2 s_val_3_4;
221
- s_val_3_4.x = reinterpret_cast<__half*>(&frag_s_3)[i];
222
- s_val_3_4.y = reinterpret_cast<__half*>(&frag_s_4)[i];
223
-
224
- frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
225
- frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
226
-}
227
-
228
-// Wait until barrier reaches `count`, then lock for current threadblock.
229
-__device__ inline void barrier_acquire(int* lock, int count) {
230
- if (threadIdx.x == 0) {
231
- int state = -1;
232
- do
233
- // Guarantee that subsequent writes by this threadblock will be visible
234
- // globally.
235
- asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
236
- : "=r"(state)
237
- : "l"(lock));
238
- while (state != count);
239
- }
240
- __syncthreads();
241
-}
242
-
243
-// Release barrier and increment visitation count.
244
-__device__ inline void barrier_release(int* lock, bool reset = false) {
245
- __syncthreads();
246
- if (threadIdx.x == 0) {
247
- if (reset) {
248
- lock[0] = 0;
249
- return;
250
- }
251
- int val = 1;
252
- // Make sure that all writes since acquiring this barrier are visible
253
- // globally, while releasing the barrier.
254
- asm volatile("fence.acq_rel.gpu;\n");
255
- asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
256
- :
257
- : "l"(lock), "r"(val));
258
- }
259
-}
260
-
41
// For a given "a" of size [M,K] performs a permutation of the K columns based
42
// on the given "perm" indices.
43
__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
@@ -335,1106 +115,6 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
115
__syncthreads();
116
}
117
338
-template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
339
- const int threads, // number of threads in a threadblock
340
- const int thread_m_blocks, // number of 16x16 blocks in the m
341
- // dimension (batchsize) of the
342
- // threadblock
343
- const int thread_n_blocks, // same for n dimension (output)
344
- const int thread_k_blocks, // same for k dimension (reduction)
345
- const int stages, // number of stages for the async global->shared
346
- // fetch pipeline
347
- const bool has_act_order, // whether act_order is enabled
348
- const int group_blocks = -1 // number of consecutive 16x16 blocks
349
- // with a separate quantization scale
350
- >
351
-__device__ inline void MarlinMoESingle(
352
- const int4* __restrict__ A, // fp16 input matrix of shape mxk
353
- const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
354
- int4* __restrict__ C, // fp16 output buffer of shape mxn
355
- const int* __restrict__ sorted_ids, // int32 sorted ids of experts
356
- const float* __restrict__ topk_weights, // float topk weights
357
- const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
358
- // (k/groupsize)xn
359
- const int* __restrict__ g_idx, // int32 group indices of shape k
360
- const int* __restrict__ expert_offsets,
361
- int num_groups, // number of scale groups per output channel
362
- int expert_idx, // idx of current expert
363
- int num_experts, // number of experts
364
- int topk, // topk parameter of moe
365
- int prob_m, // batch dimension m
366
- int prob_n, // output dimension n
367
- int prob_k, // reduction dimension k
368
- int tot_m, // total number of rows in A and C
369
- int* locks, // extra global storage for barrier synchronization
370
- bool replicate_input, // do we use the same input for each expert?
371
- bool apply_weights, // apply weights to output
372
- int current_m_block // current m block to start kernel computation from
373
-) {
374
- static constexpr auto w_type = vllm::ScalarType::from_id(w_type_id);
375
- constexpr int pack_factor = 32 / w_type.size_bits();
376
-
377
- // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
378
- // better partitioning with less reductions
379
- int parallel = 1;
380
- if (prob_m > 16 * thread_m_blocks) {
381
- parallel = prob_m / (16 * thread_m_blocks);
382
- prob_m = 16 * thread_m_blocks;
383
- }
384
-
385
- int k_tiles = prob_k / 16 / thread_k_blocks;
386
- int n_tiles = prob_n / 16 / thread_n_blocks;
387
- int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
388
-
389
- if constexpr (!has_act_order && group_blocks != -1) {
390
- if (group_blocks >= thread_k_blocks) {
391
- // Ensure that the number of tiles in each stripe is a multiple of the
392
- // groupsize; this avoids an annoying special case where a stripe starts
393
- // in the middle of group.
394
- iters = (group_blocks / thread_k_blocks) *
395
- ceildiv(iters, (group_blocks / thread_k_blocks));
396
- }
397
- }
398
-
399
- int slice_row = (iters * blockIdx.x) % k_tiles;
400
- int slice_col_par = (iters * blockIdx.x) / k_tiles;
401
- int slice_col = slice_col_par;
402
- int slice_iters; // number of threadblock tiles in the current slice
403
- int slice_count =
404
- 0; // total number of active threadblocks in the current slice
405
- int slice_idx; // index of threadblock in current slice; numbered bottom to
406
- // top
407
-
408
- // We can easily implement parallel problem execution by just remapping
409
- // indices and advancing global pointers
410
- if (slice_col_par >= n_tiles) {
411
- locks += (slice_col_par / n_tiles) * n_tiles;
412
- slice_col = slice_col_par % n_tiles;
413
- sorted_ids += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
414
- }
415
-
416
- // Compute all information about the current slice which is required for
417
- // synchronization.
418
- auto init_slice = [&]() {
419
- slice_iters =
420
- iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
421
- if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
422
- if (slice_iters == 0) return;
423
- if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
424
- slice_count = 1;
425
- slice_idx = 0;
426
- int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
427
- if (col_first <= k_tiles * (slice_col_par + 1)) {
428
- int col_off = col_first - k_tiles * slice_col_par;
429
- slice_count = ceildiv(k_tiles - col_off, iters);
430
- if (col_off > 0) slice_count++;
431
- int delta_first = iters * blockIdx.x - col_first;
432
- if (delta_first < 0 || (col_off == 0 && delta_first == 0))
433
- slice_idx = slice_count - 1;
434
- else {
435
- slice_idx = slice_count - 1 - delta_first / iters;
436
- if (col_off > 0) slice_idx--;
437
- }
438
- }
439
- if (slice_col == n_tiles) {
440
- sorted_ids += 16 * thread_m_blocks;
441
- locks += n_tiles;
442
- slice_col = 0;
443
- }
444
- };
445
- init_slice();
446
-
447
- // A sizes/strides
448
-
449
- // stride of the A matrix in global memory
450
- int a_gl_stride = prob_k / 8;
451
- // stride of an A matrix tile in shared memory
452
- constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
453
- // delta between subsequent A tiles in global memory
454
- constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
455
- // between subsequent accesses within a tile
456
- int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
457
- // between shared memory writes
458
- constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
459
- // between shared memory tile reads
460
- constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
461
- // within a shared memory tile
462
- constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
463
- // overall size of a tile
464
- constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
465
- // number of shared write iterations for a tile
466
- constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
467
-
468
- // B sizes/strides
469
- int b_gl_stride = 16 * prob_n / (pack_factor * 4);
470
- constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
471
- constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2;
472
- constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
473
-
474
- int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
475
- int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
476
- constexpr int b_sh_wr_delta = threads * b_thread_vecs;
477
- constexpr int b_sh_rd_delta = threads * b_thread_vecs;
478
- constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
479
- constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
480
-
481
- // Scale sizes/strides without act_order
482
- int s_gl_stride = prob_n / 8;
483
- constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
484
- constexpr int s_tb_groups =
485
- !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks
486
- ? thread_k_blocks / group_blocks
487
- : 1;
488
- constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
489
- int s_gl_rd_delta = s_gl_stride;
490
- // Scale size/strides with act_order
491
- constexpr int tb_k = 16 * thread_k_blocks;
492
- constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
493
- // constexpr int act_s_row_stride = 1;
494
- // int act_s_col_stride = act_s_row_stride * num_groups;
495
- int act_s_col_stride = 1;
496
- int act_s_col_warp_stride = act_s_col_stride * 8;
497
- int tb_n_warps = thread_n_blocks / 4;
498
- int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
499
-
500
- constexpr int sorted_sh_stride = threads;
501
- constexpr int sorted_gl_stride = threads;
502
-
503
- // Global A read index of current thread.
504
- int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
505
- (threadIdx.x % a_gl_rd_delta_o);
506
- a_gl_rd += a_gl_rd_delta_o * slice_row;
507
- // Shared write index of current thread.
508
- int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
509
- (threadIdx.x % a_gl_rd_delta_o);
510
- // Shared read index.
511
- int a_sh_rd =
512
- a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
513
- a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
514
-
515
- int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
516
- (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
517
- b_gl_rd += b_sh_stride * slice_col;
518
- b_gl_rd += b_gl_rd_delta_o * slice_row;
519
- int b_sh_wr = threadIdx.x * b_thread_vecs;
520
- int b_sh_rd = threadIdx.x * b_thread_vecs;
521
-
522
- // For act_order
523
- constexpr int k_iter_size = tb_k / b_sh_wr_iters;
524
- int slice_k_start = tb_k * slice_row;
525
- int slice_k_finish = slice_k_start + tb_k * slice_iters;
526
- int slice_k_start_shared_fetch = slice_k_start;
527
- int slice_n_offset = act_s_col_tb_stride * slice_col;
528
-
529
- // No act_order
530
- int s_gl_rd;
531
- if constexpr (!has_act_order) {
532
- if constexpr (group_blocks == -1) {
533
- s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
534
- } else {
535
- s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
536
- s_sh_stride * slice_col + threadIdx.x;
537
- }
538
- }
539
- int s_sh_wr = threadIdx.x;
540
- bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
541
-
542
- // We use a different scale layout for grouped and column-wise quantization as
543
- // we scale a `half2` tile in column-major layout in the former and in
544
- // row-major in the latter case.
545
- int s_sh_rd;
546
- if constexpr (group_blocks != -1)
547
- s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
548
- (threadIdx.x % 32) / 4;
549
- else
550
- s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
551
- (threadIdx.x % 32) % 4;
552
-
553
- int sh_first_group_id = -1;
554
- int sh_num_groups = -1;
555
- constexpr int sh_max_num_groups = 32;
556
-
557
- int shs_size;
558
- if constexpr (has_act_order)
559
- shs_size = sh_max_num_groups * s_sh_stride + threads;
560
- else
561
- shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
562
-
563
- extern __shared__ int4 sh[];
564
- // Shared memory storage for global fetch pipelines.
565
- int4* sh_a = sh;
566
- int4* sh_b = sh_a + (stages * a_sh_stage);
567
- int4* sh_g_idx = sh_b + (stages * b_sh_stage);
568
- int4* sh_s = sh_g_idx + (stages * g_idx_stage);
569
- int* sh_sorted = (int*)(sh_s + shs_size);
570
-
571
- // Precompute which thread should not read memory in which iterations; this is
572
- // needed if there are more threads than required for a certain tilesize or
573
- // when the batchsize is not a multiple of 16.
574
- bool a_sh_wr_pred[a_sh_wr_iters];
575
- #pragma unroll
576
- for (int i = 0; i < a_sh_wr_iters; i++) {
577
- int a_idx = a_sh_wr_delta * i + a_sh_wr;
578
- int row = a_idx / a_gl_rd_delta_o;
579
- if (row >= prob_m) {
580
- a_sh_wr_pred[i] = false;
581
- } else {
582
- a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
583
- }
584
- }
585
-
586
- // To ensure that writing and reading A tiles to/from shared memory, the
587
- // latter in fragment format, is fully bank conflict free, we need to use a
588
- // rather fancy XOR-based layout. The key here is that neither reads nor
589
- // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
590
- // same shared memory banks. Further, it seems (based on NSight-Compute) that
591
- // each warp must also write a consecutive memory segment?
592
- auto transform_a = [&](int i) {
593
- int row = i / a_gl_rd_delta_o;
594
- return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
595
- };
596
- // Since the computation of this remapping is non-trivial and, due to our main
597
- // loop unrolls, all shared memory accesses are static, we simply precompute
598
- // both transformed reads and writes.
599
- int a_sh_wr_trans[a_sh_wr_iters];
600
- #pragma unroll
601
- for (int i = 0; i < a_sh_wr_iters; i++)
602
- a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
603
- int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
604
- #pragma unroll
605
- for (int i = 0; i < b_sh_wr_iters; i++) {
606
- #pragma unroll
607
- for (int j = 0; j < thread_m_blocks; j++)
608
- a_sh_rd_trans[i][j] =
609
- transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
610
- }
611
-
612
- // Since B-accesses have non-constant stride they have to be computed at
613
- // runtime; we break dependencies between subsequent accesses with a tile by
614
- // maintining multiple pointers (we have enough registers), a tiny
615
- // optimization.
616
- const int4* B_ptr[b_sh_wr_iters];
617
- #pragma unroll
618
- for (int i = 0; i < b_sh_wr_iters; i++)
619
- B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
620
-
621
- // Register storage for double buffer of shared memory reads.
622
- FragA frag_a[2][thread_m_blocks];
623
- I4 frag_b_quant[2][b_thread_vecs];
624
- FragC frag_c[thread_m_blocks][4][2];
625
- FragS frag_s[2][4]; // No act-order
626
- FragS act_frag_s[2][4][4]; // For act-order
627
-
628
- // Zero accumulators.
629
- auto zero_accums = [&]() {
630
- #pragma unroll
631
- for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
632
- reinterpret_cast<float*>(frag_c)[i] = 0;
633
- };
634
-
635
- auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
636
- int last_group_id) {
637
- sh_first_group_id = first_group_id;
638
- sh_num_groups = last_group_id - first_group_id + 1;
639
-
640
- if (sh_num_groups < sh_max_num_groups) {
641
- sh_num_groups = sh_max_num_groups;
642
- }
643
-
644
- if (sh_first_group_id + sh_num_groups > num_groups) {
645
- sh_num_groups = num_groups - sh_first_group_id;
646
- }
647
-
648
- int row_offset = first_group_id * s_gl_stride;
649
-
650
- if (is_async) {
651
- for (int i = 0; i < sh_num_groups; i++) {
652
- if (threadIdx.x < s_sh_stride) {
653
- cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
654
- &scales_ptr[row_offset + (i * s_gl_stride) +
655
- slice_n_offset + threadIdx.x]);
656
- }
657
- }
658
- } else {
659
- for (int i = 0; i < sh_num_groups; i++) {
660
- if (threadIdx.x < s_sh_stride) {
661
- sh_s[(i * s_sh_stride) + threadIdx.x] =
662
- scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
663
- threadIdx.x];
664
- }
665
- }
666
- }
667
- };
668
- // Asynchronously fetch the next A, B and s tile from global to the next
669
- // shared memory pipeline location.
670
- auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
671
- if (pred) {
672
- int4* sh_a_stage = sh_a + a_sh_stage * pipe;
673
- #pragma unroll
674
- for (int i = 0; i < a_sh_wr_iters; i++) {
675
- int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off;
676
- int row = a_idx / a_gl_stride;
677
- int sorted_row =
678
- replicate_input ? sorted_ids[row] / topk : sorted_ids[row];
679
- int new_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride;
680
- if (sorted_row < tot_m * (replicate_input ? 1 : topk) &&
681
- new_idx < a_gl_stride * tot_m * (replicate_input ? 1 : topk)) {
682
- cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[new_idx],
683
- a_sh_wr_pred[i]);
684
- }
685
- }
686
- int4* sh_b_stage = sh_b + b_sh_stage * pipe;
687
- #pragma unroll
688
- for (int i = 0; i < b_sh_wr_iters; i++) {
689
- #pragma unroll
690
- for (int j = 0; j < b_thread_vecs; j++) {
691
- cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
692
- }
693
- B_ptr[i] += b_gl_rd_delta_o;
694
- }
695
-
696
- if constexpr (has_act_order) {
697
- // Fetch g_idx thread-block portion
698
- int full_pipe = a_off;
699
- int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
700
- if (cur_k < prob_k && cur_k < slice_k_finish) {
701
- int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
702
-
703
- int4 const* cur_g_idx_stage_ptr =
704
- reinterpret_cast<int4 const*>(&g_idx[cur_k]);
705
-
706
- if (threadIdx.x < g_idx_stage) {
707
- cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
708
- &cur_g_idx_stage_ptr[threadIdx.x]);
709
- }
710
- }
711
- } else {
712
- if constexpr (group_blocks != -1) {
713
- int4* sh_s_stage = sh_s + s_sh_stage * pipe;
714
-
715
- if constexpr (group_blocks >= thread_k_blocks) {
716
- // Only fetch scales if this tile starts a new group
717
- if (pipe % (group_blocks / thread_k_blocks) == 0) {
718
- if (s_sh_wr_pred) {
719
- cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
720
- }
721
- s_gl_rd += s_gl_rd_delta;
722
- }
723
- } else {
724
- for (int i = 0; i < s_tb_groups; i++) {
725
- if (s_sh_wr_pred) {
726
- cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
727
- &scales_ptr[s_gl_rd]);
728
- }
729
- s_gl_rd += s_gl_rd_delta;
730
- }
731
- }
732
- }
733
- }
734
- }
735
- // Insert a fence even when we are winding down the pipeline to ensure that
736
- // waiting is also correct at this point.
737
- cp_async_fence();
738
- };
739
-
740
- // TODO we are currently hitting illegal memory accesses when fetching
741
- // sorted_ids to shared data: fix this
742
- auto fetch_sorted_ids_to_shared = [&]() {
743
- const int mpt = ceildiv(prob_m, threads);
744
- for (int i = 0; i < mpt; i++) {
745
- if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
746
- sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
747
- sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
748
- }
749
- }
750
- };
751
-
752
- // Wait until the next thread tile has been loaded to shared memory.
753
- auto wait_for_stage = [&]() {
754
- // We only have `stages - 2` active fetches since we are double buffering
755
- // and can only issue the next fetch when it is guaranteed that the previous
756
- // shared memory load is fully complete (as it may otherwise be
757
- // overwritten).
758
- cp_async_wait<stages - 2>();
759
- __syncthreads();
760
- };
761
-
762
- // Load the next sub-tile from the current location in the shared memory pipe
763
- // into the current register buffer.
764
- auto fetch_to_registers = [&](int k, int pipe) {
765
- int4* sh_a_stage = sh_a + a_sh_stage * pipe;
766
- #pragma unroll
767
- for (int i = 0; i < thread_m_blocks; i++)
768
- ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
769
- int4* sh_b_stage = sh_b + b_sh_stage * pipe;
770
-
771
- #pragma unroll
772
- for (int i = 0; i < b_thread_vecs; i++) {
773
- frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
774
- &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
775
- }
776
- };
777
-
778
- bool is_same_group[stages];
779
- int same_group_id[stages];
780
-
781
- auto init_same_group = [&](int pipe) {
782
- if constexpr (!has_act_order) {
783
- is_same_group[pipe] = false;
784
- same_group_id[pipe] = 0;
785
- return;
786
- }
787
-
788
- int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
789
- int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
790
-
791
- int group_id_1 = sh_g_idx_int_ptr[0];
792
- int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
793
-
794
- is_same_group[pipe] = group_id_1 == group_id_2;
795
- same_group_id[pipe] = group_id_1;
796
- };
797
-
798
- auto fetch_scales_to_registers = [&](int k, int full_pipe) {
799
- int pipe = full_pipe % stages;
800
-
801
- if constexpr (!has_act_order) {
802
- // No act-order case
803
- if constexpr (group_blocks != -1) {
804
- if constexpr (group_blocks >= thread_k_blocks) {
805
- int4* sh_s_stage =
806
- sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
807
- (pipe / (group_blocks / thread_k_blocks)));
808
- reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
809
- } else {
810
- int warp_id = threadIdx.x / 32;
811
- int n_warps = thread_n_blocks / 4;
812
-
813
- int warp_row = warp_id / n_warps;
814
-
815
- int cur_k = warp_row * 16;
816
- cur_k += k_iter_size * (k % b_sh_wr_iters);
817
-
818
- int k_blocks = cur_k / 16;
819
- int cur_group_id = k_blocks / group_blocks;
820
-
821
- int4* sh_s_stage = sh_s + s_sh_stage * pipe;
822
-
823
- reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
824
- sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
825
- }
826
- }
827
-
828
- return;
829
- }
830
-
831
- // Act-order case
832
-
833
- // Determine K of the "current" thread-block
834
- int cur_k = slice_k_start + tb_k * full_pipe;
835
- if (cur_k >= prob_k || cur_k >= slice_k_finish) {
836
- return;
837
- }
838
-
839
- // Reset (to current thread-block) since we read g_idx portion from the
840
- // shared memory
841
- cur_k = 0;
842
-
843
- // Progress to current iteration
844
- cur_k += k_iter_size * (k % b_sh_wr_iters);
845
-
846
- // Determine "position" inside the thread-block (based on warp and
847
- // thread-id)
848
- int warp_id = threadIdx.x / 32;
849
- int n_warps =
850
- thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
851
-
852
- int warp_row = warp_id / n_warps;
853
- int warp_col = warp_id % n_warps;
854
-
855
- cur_k += warp_row * 16;
856
-
857
- int th_id = threadIdx.x % 32;
858
- cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
859
-
860
- int s_col_shift =
861
- /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
862
- (th_id / 4) * act_s_col_stride;
863
-
864
- if (is_same_group[pipe]) {
865
- if (k % 2 == 0) {
866
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
867
- sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
868
- s_col_shift];
869
- } else {
870
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
871
- *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
872
- }
873
-
874
- for (int i = 1; i < 4; i++) {
875
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
876
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
877
- }
878
- return;
879
- }
880
-
881
- int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
882
- int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
883
-
884
- constexpr int k_frag_offsets[4] = {0, 1, 8,
885
- 9}; // Tensor core offsets per thread
886
-
887
- #pragma unroll
888
- for (int i = 0; i < 4; i++) {
889
- int actual_k = cur_k + k_frag_offsets[i];
890
-
891
- int group_id = sh_g_idx_int_ptr[actual_k];
892
- int rel_group_id = group_id - sh_first_group_id;
893
-
894
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
895
- sh_s[rel_group_id * s_sh_stride + s_col_shift];
896
- }
897
- };
898
-
899
- // Execute the actual tensor core matmul of a sub-tile.
900
- auto matmul = [&](int k) {
901
- // We have the m dimension as the inner loop in order to encourage overlapping
902
- // dequantization and matmul operations.
903
- #pragma unroll
904
- for (int j = 0; j < 4; j++) {
905
- int b_quant_0, b_quant_1;
906
- if constexpr (w_type.size_bits() == 4) {
907
- b_quant_0 = frag_b_quant[k % 2][0][j];
908
- b_quant_1 = b_quant_0 >> 8;
909
- } else {
910
- static_assert(w_type.size_bits() == 8);
911
- int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
912
- b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
913
- b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
914
- }
915
-
916
- FragB frag_b0 = dequant<w_type_id>(b_quant_0);
917
- FragB frag_b1 = dequant<w_type_id>(b_quant_1);
918
-
919
- // Apply scale to frag_b0
920
- if constexpr (has_act_order) {
921
- scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
922
- act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
923
- } else {
924
- if constexpr (group_blocks != -1) {
925
- scale(frag_b0, frag_s[k % 2][j], 0);
926
- }
927
- }
928
-
929
- // Apply scale to frag_b1
930
- if constexpr (has_act_order) {
931
- scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
932
- act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
933
-
934
- } else {
935
- if constexpr (group_blocks != -1) {
936
- scale(frag_b1, frag_s[k % 2][j], 1);
937
- }
938
- }
939
-
940
- #pragma unroll
941
- for (int i = 0; i < thread_m_blocks; i++) {
942
- mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
943
- mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
944
- }
945
- }
946
- };
947
-
948
- // Since we slice across the k dimension of a tile in order to increase the
949
- // number of warps while keeping the n dimension of a tile reasonable, we have
950
- // multiple warps that accumulate their partial sums of the same output
951
- // location; which we have to reduce over in the end. We do in shared memory.
952
- auto thread_block_reduce = [&]() {
953
- constexpr int red_off = threads / b_sh_stride_threads / 2;
954
- if (red_off >= 1) {
955
- int red_idx = threadIdx.x / b_sh_stride_threads;
956
- constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
957
- constexpr int red_sh_delta = b_sh_stride_threads;
958
- int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
959
- (threadIdx.x % b_sh_stride_threads);
960
-
961
- // Parallel logarithmic shared memory reduction. We make sure to avoid any
962
- // unnecessary read or write iterations, e.g., for two warps we write only
963
- // once by warp 1 and read only once by warp 0.
964
-
965
- #pragma unroll
966
- for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
967
- #pragma unroll
968
- for (int i = red_off; i > 0; i /= 2) {
969
- if (i <= red_idx && red_idx < 2 * i) {
970
- #pragma unroll
971
- for (int j = 0; j < 4 * 2; j++) {
972
- int red_sh_wr =
973
- red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
974
- if (i < red_off) {
975
- float* c_rd =
976
- reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
977
- float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
978
- #pragma unroll
979
- for (int k = 0; k < 4; k++)
980
- reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
981
- c_rd[k] + c_wr[k];
982
- }
983
- sh[red_sh_wr] =
984
- reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
985
- }
986
- }
987
- __syncthreads();
988
- }
989
- if (red_idx == 0) {
990
- #pragma unroll
991
- for (int i = 0; i < 4 * 2; i++) {
992
- float* c_rd =
993
- reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
994
- #pragma unroll
995
- for (int j = 0; j < 4; j++)
996
- reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
997
- c_rd[j];
998
- }
999
- }
1000
- __syncthreads();
1001
- }
1002
- }
1003
- };
1004
-
1005
- // Since multiple threadblocks may process parts of the same column slice, we
1006
- // finally have to globally reduce over the results. As the striped
1007
- // partitioning minimizes the number of such reductions and our outputs are
1008
- // usually rather small, we perform this reduction serially in L2 cache.
1009
- auto global_reduce = [&](bool first = false, bool last = false) {
1010
- // We are very careful here to reduce directly in the output buffer to
1011
- // maximize L2 cache utilization in this step. To do this, we write out
1012
- // results in FP16 (but still reduce with FP32 compute).
1013
- constexpr int active_threads = 32 * thread_n_blocks / 4;
1014
- if (threadIdx.x < active_threads) {
1015
- int c_gl_stride = prob_n / 8;
1016
- int c_gl_wr_delta_o = 8 * c_gl_stride;
1017
- int c_gl_wr_delta_i = 4 * (active_threads / 32);
1018
- int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
1019
- 4 * (threadIdx.x / 32) + threadIdx.x % 4;
1020
- c_gl_wr += (2 * thread_n_blocks) * slice_col;
1021
- constexpr int c_sh_wr_delta = active_threads;
1022
- int c_sh_wr = threadIdx.x;
1023
-
1024
- int row = (threadIdx.x % 32) / 4;
1025
-
1026
- if (!first) {
1027
- // Interestingly, doing direct global accesses here really seems to mess up
1028
- // the compiler and lead to slowdowns, hence we also use async-copies even
1029
- // though these fetches are not actually asynchronous.
1030
- #pragma unroll
1031
- for (int i = 0; i < thread_m_blocks * 4; i++) {
1032
- int c_idx =
1033
- c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
1034
- int sorted_row = sorted_ids[c_idx / c_gl_stride];
1035
- int new_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride;
1036
- cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i], &C[new_idx],
1037
- sorted_row < tot_m * topk &&
1038
- (8 * (i / 2) + row < prob_m &&
1039
- (i < (thread_m_blocks - 1) * 4 ||
1040
- sorted_ids[8 * (i / 2) + row] < tot_m * topk)));
1041
- }
1042
- cp_async_fence();
1043
- cp_async_wait<0>();
1044
- }
1045
-
1046
- #pragma unroll
1047
- for (int i = 0; i < thread_m_blocks * 4; i++) {
1048
- if (8 * (i / 2) + row < prob_m &&
1049
- (i < (thread_m_blocks - 1) * 4 ||
1050
- sorted_ids[8 * (i / 2) + row] < tot_m * topk)) {
1051
- if (!first) {
1052
- int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
1053
- #pragma unroll
1054
- for (int j = 0; j < 2 * 4; j++) {
1055
- reinterpret_cast<float*>(
1056
- &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
1057
- __half2float(reinterpret_cast<__half*>(&c_red)[j]);
1058
- }
1059
- }
1060
- if (!last) {
1061
- int4 c;
1062
- #pragma unroll
1063
- for (int j = 0; j < 2 * 4; j++) {
1064
- reinterpret_cast<__half*>(&c)[j] =
1065
- __float2half(reinterpret_cast<float*>(
1066
- &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
1067
- }
1068
- int c_idx =
1069
- c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
1070
- int row = sorted_ids[c_idx / c_gl_stride];
1071
- if (row < tot_m * topk) {
1072
- int new_idx = row * c_gl_stride + c_idx % c_gl_stride;
1073
- C[new_idx] = c;
1074
- }
1075
- }
1076
- }
1077
- }
1078
- }
1079
- };
1080
-
1081
- // Write out the reduce final result in the correct layout. We only actually
1082
- // reshuffle matrix fragments in this step, the reduction above is performed
1083
- // in fragment layout.
1084
- auto write_result = [&]() {
1085
- int c_gl_stride = prob_n / 8;
1086
- constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
1087
- int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
1088
- constexpr int c_sh_rd_delta =
1089
- c_sh_stride * (threads / (2 * thread_n_blocks));
1090
-
1091
- int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1092
- (threadIdx.x % (2 * thread_n_blocks));
1093
- c_gl_wr += (2 * thread_n_blocks) * slice_col;
1094
- int c_sh_wr =
1095
- (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
1096
- c_sh_wr += 32 * (threadIdx.x / 32);
1097
- int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1098
- (threadIdx.x % (2 * thread_n_blocks));
1099
-
1100
- int c_gl_wr_end = c_gl_stride * prob_m;
1101
-
1102
- // We first reorder in shared memory to guarantee the most efficient final
1103
- // global write patterns
1104
- auto write = [&](int idx, float c0, float c1, FragS& s) {
1105
- half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1106
-
1107
- // For per-column quantization we finally apply the scale here (only for
1108
- // 4-bit)
1109
- if constexpr (!has_act_order && group_blocks == -1 &&
1110
- w_type.size_bits() == 4) {
1111
- res = __hmul2(res, s[0]);
1112
- }
1113
-
1114
- ((half2*)sh)[idx] = res;
1115
- };
1116
- if (threadIdx.x / 32 < thread_n_blocks / 4) {
1117
- #pragma unroll
1118
- for (int i = 0; i < thread_m_blocks; i++) {
1119
- #pragma unroll
1120
- for (int j = 0; j < 4; j++) {
1121
- int wr = c_sh_wr + 8 * j;
1122
- write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
1123
- frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
1124
- write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
1125
- frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
1126
- write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
1127
- frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
1128
- write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
1129
- frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
1130
- }
1131
- c_sh_wr += 16 * (4 * c_sh_stride);
1132
- }
1133
- }
1134
- __syncthreads();
1135
-
1136
- #pragma unroll
1137
- for (int i = 0;
1138
- i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1139
- i++) {
1140
- if (c_gl_wr < c_gl_wr_end) {
1141
- int row = sorted_ids[c_gl_wr / c_gl_stride];
1142
- if (row < tot_m * topk) {
1143
- int off = row * c_gl_stride + c_gl_wr % c_gl_stride;
1144
- if (!apply_weights) {
1145
- C[off] = sh[c_sh_rd];
1146
- } else {
1147
- __half* ctrg = reinterpret_cast<__half*>(&C[off]);
1148
- __half* csrc = reinterpret_cast<__half*>(&sh[c_sh_rd]);
1149
- for (int j = 0; j < 8; ++j) {
1150
- ctrg[j] = __float2half(topk_weights[row] * __half2float(csrc[j]));
1151
- }
1152
- }
1153
- c_gl_wr += c_gl_wr_delta;
1154
- c_sh_rd += c_sh_rd_delta;
1155
- }
1156
- }
1157
- }
1158
- };
1159
-
1160
- // Start global fetch and register load pipelines.
1161
- auto start_pipes = [&]() {
1162
- // TODO re-enable after fixing this function
1163
- // fetch_sorted_ids_to_shared();
1164
- // __syncthreads();
1165
-
1166
- #pragma unroll
1167
- for (int i = 0; i < stages - 1; i++) {
1168
- if (has_act_order && i == 0) {
1169
- int last_g_idx = slice_k_start + stages * tb_k * 2;
1170
- if (last_g_idx >= prob_k) {
1171
- last_g_idx = prob_k - 1;
1172
- }
1173
- fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1174
- }
1175
- fetch_to_shared(i, i, i < slice_iters);
1176
- }
1177
-
1178
- zero_accums();
1179
- wait_for_stage();
1180
- init_same_group(0);
1181
- fetch_to_registers(0, 0);
1182
- fetch_scales_to_registers(0, 0);
1183
- a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1184
- slice_k_start_shared_fetch += tb_k * (stages - 1);
1185
- };
1186
- if (slice_iters) {
1187
- start_pipes();
1188
- }
1189
-
1190
- // Main loop.
1191
- while (slice_iters) {
1192
- // We unroll over both the global fetch and the register load pipeline to
1193
- // ensure all shared memory accesses are static. Note that both pipelines
1194
- // have even length meaning that the next iteration will always start at
1195
- // index 0.
1196
- #pragma unroll
1197
- for (int pipe = 0; pipe < stages;) {
1198
- #pragma unroll
1199
- for (int k = 0; k < b_sh_wr_iters; k++) {
1200
- fetch_to_registers(k + 1, pipe % stages);
1201
- fetch_scales_to_registers(k + 1, pipe);
1202
- if (k == b_sh_wr_iters - 2) {
1203
- fetch_to_shared((pipe + stages - 1) % stages, pipe,
1204
- slice_iters >= stages);
1205
- pipe++;
1206
- wait_for_stage();
1207
- init_same_group(pipe % stages);
1208
- }
1209
- matmul(k);
1210
- }
1211
- slice_iters--;
1212
- if (slice_iters == 0) {
1213
- break;
1214
- }
1215
- }
1216
-
1217
- a_gl_rd += a_gl_rd_delta_o * stages;
1218
- slice_k_start += tb_k * stages;
1219
- slice_k_start_shared_fetch += tb_k * stages;
1220
-
1221
- if constexpr (has_act_order) {
1222
- int first_group_id = g_idx[slice_k_start];
1223
- int last_g_idx = slice_k_start + stages * tb_k * 2;
1224
- if (last_g_idx >= prob_k) {
1225
- last_g_idx = prob_k - 1;
1226
- }
1227
- int last_group_id = g_idx[last_g_idx];
1228
- if (last_group_id >= sh_first_group_id + sh_num_groups) {
1229
- fetch_scales_to_shared(false, first_group_id, last_group_id);
1230
- __syncthreads();
1231
- }
1232
- }
1233
-
1234
- // Process results and, if necessary, proceed to the next column slice.
1235
- // While this pattern may not be the most readable, other ways of writing
1236
- // the loop seemed to noticeably worse performance after compilation.
1237
- if (slice_iters == 0) {
1238
- cp_async_wait<0>();
1239
- bool last = slice_idx == slice_count - 1;
1240
- if constexpr (!has_act_order && group_blocks == -1) {
1241
- if constexpr (w_type.size_bits() == 8) {
1242
- if (s_sh_wr_pred) {
1243
- cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1244
- }
1245
- cp_async_fence();
1246
- } else {
1247
- // For 4-bit per-column scales, we only fetch them here in the
1248
- // final step before write-out
1249
- if (last) {
1250
- if (s_sh_wr_pred) {
1251
- cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1252
- }
1253
- cp_async_fence();
1254
- }
1255
- }
1256
- }
1257
-
1258
- thread_block_reduce();
1259
- if constexpr (!has_act_order && group_blocks == -1) {
1260
- if constexpr (w_type.size_bits() == 8) {
1261
- cp_async_wait<0>();
1262
- __syncthreads();
1263
- if (threadIdx.x / 32 < thread_n_blocks / 4) {
1264
- reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1265
- reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1266
- }
1267
-
1268
- } else {
1269
- if (last) {
1270
- cp_async_wait<0>();
1271
- __syncthreads();
1272
- if (threadIdx.x / 32 < thread_n_blocks / 4) {
1273
- reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1274
- reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1275
- }
1276
- }
1277
- }
1278
- }
1279
-
1280
- // For 8-bit channelwise, we apply the scale before the global reduction
1281
- // that converts the fp32 results to fp16 (so that we avoid possible
1282
- // overflow in fp16)
1283
- if constexpr (!has_act_order && group_blocks == -1 &&
1284
- w_type.size_bits() == 8) {
1285
- if (threadIdx.x / 32 < thread_n_blocks / 4) {
1286
- #pragma unroll
1287
- for (int i = 0; i < thread_m_blocks; i++) {
1288
- #pragma unroll
1289
- for (int j = 0; j < 4; j++) {
1290
- scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][0]),
1291
- frag_s[j / 2][2 * (j % 2) + 0]);
1292
- scale_float(reinterpret_cast<float*>(&frag_c[i][j][0][2]),
1293
- frag_s[j / 2][2 * (j % 2) + 0]);
1294
-
1295
- scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][0]),
1296
- frag_s[j / 2][2 * (j % 2) + 1]);
1297
- scale_float(reinterpret_cast<float*>(&frag_c[i][j][1][2]),
1298
- frag_s[j / 2][2 * (j % 2) + 1]);
1299
- }
1300
- }
1301
- }
1302
- }
1303
-
1304
- if (slice_count > 1) { // only globally reduce if there is more than one
1305
- // block in a slice
1306
- barrier_acquire(&locks[slice_col], slice_idx);
1307
- global_reduce(slice_idx == 0, last);
1308
- barrier_release(&locks[slice_col], last);
1309
- }
1310
- if (last) // only the last block in a slice actually writes the result
1311
- write_result();
1312
- slice_row = 0;
1313
- slice_col_par++;
1314
- slice_col++;
1315
- init_slice();
1316
- if (slice_iters) {
1317
- a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1318
- (threadIdx.x % a_gl_rd_delta_o);
1319
- #pragma unroll
1320
- for (int i = 0; i < b_sh_wr_iters; i++)
1321
- B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1322
- if (slice_col == 0) {
1323
- #pragma unroll
1324
- for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1325
- }
1326
-
1327
- // Update slice k/n for scales loading
1328
- if constexpr (has_act_order) {
1329
- slice_k_start = tb_k * slice_row;
1330
- slice_k_finish = slice_k_start + tb_k * slice_iters;
1331
- slice_k_start_shared_fetch = slice_k_start;
1332
- slice_n_offset = act_s_col_tb_stride * slice_col;
1333
-
1334
- } else {
1335
- s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1336
- }
1337
- start_pipes();
1338
- }
1339
- }
1340
- }
1341
-}
1342
-
1343
-template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1344
- const int threads, // number of threads in a threadblock
1345
- const int thread_n_blocks, // same for n dimension (output)
1346
- const int thread_k_blocks, // same for k dimension (reduction)
1347
- const int stages, // number of stages for the async global->shared
1348
- // fetch pipeline
1349
- const bool has_act_order, // whether act_order is enabled
1350
- const int group_blocks = -1 // number of consecutive 16x16 blocks
1351
- // with a separate quantization scale
1352
- >
1353
-__global__ void MarlinMoE(
1354
- const int4* __restrict__ A, // fp16 input matrix of shape mxk
1355
- const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1356
- int4* __restrict__ C, // fp16 output buffer of shape mxn
1357
- const int* __restrict__ sorted_ids_base, // int32 sorted ids of experts
1358
- const float* __restrict__ topk_weights, // float topk weights
1359
- const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1360
- // (k/groupsize)xn
1361
- const int* __restrict__ g_idx, // int32 group indices of shape k
1362
- const int* __restrict__ expert_offsets,
1363
- int num_groups, // number of scale groups per output channel
1364
- int expert_idx, // idx of current expert
1365
- int num_experts, // number of experts
1366
- int topk, // topk parameter of moe
1367
- int prob_m, // batch dimension m
1368
- int prob_n, // output dimension n
1369
- int prob_k, // reduction dimension k
1370
- int tot_m, // total number of rows in A and C
1371
- int* locks, // extra global storage for barrier synchronization
1372
- bool replicate_input, // do we use the same input for each expert?
1373
- bool apply_weights, // apply weights to output
1374
- int current_m_block, // current m block to start kernel computation from
1375
- int max_par, // maximum parallelism
1376
- int cfg_max_m_blocks // upper bound on m blocks
1377
-) {
1378
- int m_block_ctr = current_m_block;
1379
-
1380
- const int* sorted_ids_expert =
1381
- sorted_ids_base + expert_offsets[expert_idx] + m_block_ctr * 4 * max_par;
1382
- int tot_its = expert_offsets[expert_idx + 1] - expert_offsets[expert_idx];
1383
- if (tot_its == 0) {
1384
- return;
1385
- }
1386
- int tot_m_blocks = ceildiv(tot_its, 16);
1387
- int pad = 16 * tot_m_blocks - tot_its;
1388
-
1389
- if (m_block_ctr >= tot_m_blocks) {
1390
- return;
1391
- }
1392
-
1393
- int max_block = tot_m_blocks - m_block_ctr;
1394
- prob_m = tot_its - 16 * m_block_ctr;
1395
-
1396
- int par = 1;
1397
- if (max_block > cfg_max_m_blocks) {
1398
- // Note that parallel > 1 currently only works for inputs without any
1399
- // padding
1400
- par = (16 * max_block - pad) / (16 * cfg_max_m_blocks);
1401
- if (par > max_par) par = max_par;
1402
- prob_m = (16 * cfg_max_m_blocks) * par;
1403
- m_block_ctr += cfg_max_m_blocks * (par - 1);
1404
- max_block = cfg_max_m_blocks;
1405
- }
1406
-
1407
- if (max_block == 1) {
1408
- MarlinMoESingle<w_type_id, threads, 1, thread_n_blocks, thread_k_blocks,
1409
- stages, has_act_order, group_blocks>(
1410
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1411
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1412
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1413
- current_m_block);
1414
- } else if (max_block == 2) {
1415
- MarlinMoESingle<w_type_id, threads, 2, thread_n_blocks, thread_k_blocks,
1416
- stages, has_act_order, group_blocks>(
1417
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1418
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1419
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1420
- current_m_block);
1421
- } else if (max_block == 3) {
1422
- MarlinMoESingle<w_type_id, threads, 3, thread_n_blocks, thread_k_blocks,
1423
- stages, has_act_order, group_blocks>(
1424
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1425
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1426
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1427
- current_m_block);
1428
- } else {
1429
- MarlinMoESingle<w_type_id, threads, 4, thread_n_blocks, thread_k_blocks,
1430
- stages, has_act_order, group_blocks>(
1431
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1432
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1433
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1434
- current_m_block);
1435
- }
1436
-}
1437
-
118
#else
119
120
__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
@@ -1454,81 +134,8 @@ __global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
134
return;
135
}
136
1457
-template <const vllm::ScalarTypeId w_type_id, // weight ScalarType id
1458
- const int threads, // number of threads in a threadblock
1459
- const int thread_n_blocks, // same for n dimension (output)
1460
- const int thread_k_blocks, // same for k dimension (reduction)
1461
- const int stages, // number of stages for the async global->shared
1462
- // fetch pipeline
1463
- const bool has_act_order, // whether act_order is enabled
1464
- const int group_blocks = -1 // number of consecutive 16x16 blocks
1465
- // with a separate quantization scale
1466
- >
1467
-__global__ void MarlinMoE(
1468
- const int4* __restrict__ A, // fp16 input matrix of shape mxk
1469
- const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1470
- int4* __restrict__ C, // fp16 output buffer of shape mxn
1471
- const int* __restrict__ sorted_ids, // int32 sorted ids of experts
1472
- const float* __restrict__ topk_weights, // float topk weights
1473
- const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1474
- // (k/groupsize)xn
1475
- const int* __restrict__ g_idx, // int32 group indices of shape k
1476
- const int* __restrict__ expert_offsets,
1477
- int num_groups, // number of scale groups per output channel
1478
- int expert_idx, // idx of current expert
1479
- int num_experts, // number of experts
1480
- int topk, // topk parameter of moe
1481
- int prob_m, // batch dimension m
1482
- int prob_n, // output dimension n
1483
- int prob_k, // reduction dimension k
1484
- int tot_m, // total number of rows in A and C
1485
- int* locks, // extra global storage for barrier synchronization
1486
- bool replicate_input, // do we use the same input for each expert?
1487
- bool apply_weights, // apply weights to output
1488
- int current_m_block, // current m block to start kernel computation from
1489
- int max_par, // maximum parallelism
1490
- int cfg_max_m_blocks // upper bound on m blocks
1491
-
1492
-) {
1493
- // Marlin is not implemented yet for SM < 8.0
1494
- assert(false);
1495
- return;
1496
-}
1497
-
137
#endif
138
1500
-// 8 warps are a good choice since every SM has 4 schedulers and having more
1501
-// than 1 warp per schedule allows some more latency hiding. At the same time,
1502
-// we want relatively few warps to have many registers per warp and small tiles.
1503
-const int USER_THREADS =
1504
- 256; // Note: This is only used with user-provided thread_k/n
1505
-const int STAGES = 4; // 4 pipeline stages fit into shared memory
1506
-// const int SHARED_MEM =
1507
-// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1508
-
1509
-static constexpr int min_thread_n = 64;
1510
-static constexpr int min_thread_k = 64;
1511
-
1512
-#define __CALL_IF_MOE(W_TYPE, THREAD_N_BLOCKS, THREAD_K_BLOCKS, HAS_ACT_ORDER, \
1513
- GROUP_BLOCKS, NUM_THREADS) \
1514
- else if (q_type == W_TYPE && thread_n_blocks == THREAD_N_BLOCKS && \
1515
- thread_k_blocks == THREAD_K_BLOCKS && \
1516
- has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1517
- num_threads == NUM_THREADS) { \
1518
- cudaFuncSetAttribute( \
1519
- MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1520
- STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1521
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1522
- MarlinMoE<W_TYPE.id(), NUM_THREADS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1523
- STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1524
- <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1525
- A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1526
- g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1527
- num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1528
- replicate_input, apply_weights, m_block, max_par, \
1529
- exec_cfg.max_m_blocks); \
1530
- }
1531
-
139
typedef struct {
140
int thread_k;
141
int thread_n;
@@ -1703,25 +310,27 @@ exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
310
return exec_config_t{0, {-1, -1, -1}};
311
}
312
1706
-#define CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1707
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1708
- \
1709
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1710
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1711
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1712
- __CALL_IF_MOE(W_TYPE, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1713
-
1714
-void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1715
- const void* sorted_ids, const void* topk_weights,
1716
- const void* topk_ids, const void* s, const void* g_idx,
1717
- const void* perm, void* a_tmp, void* expert_offsets,
1718
- int prob_m, int prob_n, int prob_k, void* workspace,
1719
- vllm::ScalarType const& q_type, bool has_act_order,
1720
- bool is_k_full, int num_groups, int group_size,
1721
- int num_experts, int topk, int moe_block_size, int dev,
1722
- cudaStream_t stream, int thread_k, int thread_n,
1723
- int sms, int max_par, bool replicate_input,
1724
- bool apply_weights) {
313
+#define CALL_MOE_KERNEL_FUNCTION(KERNEL_FUNCTION) \
314
+ else if (KERNEL_FUNCTION(q_type, thread_n_blocks, thread_k_blocks, \
315
+ has_act_order, group_blocks, num_threads, blocks, \
316
+ max_shared_mem, stream, A_ptr, B_ptr, C_ptr, \
317
+ sorted_ids_ptr, topk_weights_ptr, s_ptr, g_idx_ptr, \
318
+ expert_offsets_ptr, num_groups, expert_idx, \
319
+ num_experts, topk, prob_m, prob_n, prob_k, tot_m, \
320
+ locks, replicate_input, apply_weights, m_block, \
321
+ max_par, exec_cfg.max_m_blocks)) { \
322
+ }
323
+
324
+void marlin_mm_moe(const void* A, const void* B, void* C,
325
+ const void* sorted_ids, const void* topk_weights,
326
+ const void* topk_ids, const void* s, const void* g_idx,
327
+ const void* perm, void* a_tmp, void* expert_offsets,
328
+ int prob_m, int prob_n, int prob_k, void* workspace,
329
+ vllm::ScalarType const& q_type, bool has_act_order,
330
+ bool is_k_full, int num_groups, int group_size,
331
+ int num_experts, int topk, int moe_block_size, int dev,
332
+ cudaStream_t stream, int thread_k, int thread_n, int sms,
333
+ int max_par, bool replicate_input, bool apply_weights) {
334
TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
335
", ", prob_n, ", ", prob_k, "]");
336
@@ -1845,26 +454,16 @@ void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
454
int tot_m_blocks = ceildiv(tot_m, 16);
455
for (int m_block = 0; m_block < tot_m_blocks;
456
m_block += 4 * exec_cfg.max_m_blocks) {
1848
- // make it max possible value
1849
- int thread_m_blocks = exec_cfg.max_m_blocks;
1850
-
457
if (false) {
458
}
1853
- CALL_IF_MOE(vllm::kU4B8, 16, 4, 256)
1854
- CALL_IF_MOE(vllm::kU4B8, 8, 8, 256)
1855
- CALL_IF_MOE(vllm::kU4B8, 8, 4, 128)
1856
- CALL_IF_MOE(vllm::kU4B8, 4, 8, 128)
1857
- CALL_IF_MOE(vllm::kU8B128, 16, 4, 256)
1858
- CALL_IF_MOE(vllm::kU8B128, 8, 8, 256)
1859
- CALL_IF_MOE(vllm::kU8B128, 8, 4, 128)
1860
- CALL_IF_MOE(vllm::kU8B128, 4, 8, 128)
459
+ CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku4b8)
460
+ CALL_MOE_KERNEL_FUNCTION(call_marlin_moe_kernel_ku8b128)
461
else {
462
TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
463
str(prob_n) + ", " + str(prob_k) + "]" +
464
", has_act_order = " + str(has_act_order) +
465
", num_groups = " + str(num_groups) +
466
", group_size = " + str(group_size) +
1867
- ", thread_m_blocks = " + str(thread_m_blocks) +
467
", thread_n_blocks = " + str(thread_n_blocks) +
468
", thread_k_blocks = " + str(thread_k_blocks));
469
}
@@ -1943,7 +542,7 @@ torch::Tensor marlin_gemm_moe(
542
}
543
}
544
1946
- marlin_moe::marlin_mm_moe_f16i4(
545
+ marlin_moe::marlin_mm_moe(
546
a.data_ptr(), b_q_weights.data_ptr(), c.data_ptr(), sorted_ids.data_ptr(),
547
topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
548
g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),