@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8678a69ab5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Kernel] Expand MoE weight loading + Add Fused Marlin MoE Kernel (#7527)

Co-authored-by: ElizaWszola <eliza@neuralmagic.com>

Dipika Sikka committed Aug 21, 2024 at 19:17 UTC 8678a69ab51956031e3bb70bdf1a781a8652e67d
15 files changed +2374 -84
CMakeLists.txt
+2 -1
@@ -286,7 +286,8 @@ define_gpu_extension_target(
286
287 set(VLLM_MOE_EXT_SRC
288 "csrc/moe/torch_bindings.cpp"
289 - "csrc/moe/topk_softmax_kernels.cu")
289 + "csrc/moe/topk_softmax_kernels.cu"
290 + "csrc/moe/marlin_moe_ops.cu")
291
292 define_gpu_extension_target(
293 _moe_C
csrc/moe/marlin_moe_ops.cu new
+1740
@@ -0,0 +1,1740 @@
1 +/*
2 + * Modified by Neural Magic
3 + * Copyright (C) Marlin.2024 Elias Frantar
4 + *
5 + * Licensed under the Apache License, Version 2.0 (the "License");
6 + * you may not use this file except in compliance with the License.
7 + * You may obtain a copy of the License at
8 + *
9 + * http://www.apache.org/licenses/LICENSE-2.0
10 + *
11 + * Unless required by applicable law or agreed to in writing, software
12 + * distributed under the License is distributed on an "AS IS" BASIS,
13 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 + * See the License for the specific language governing permissions and
15 + * limitations under the License.
16 + */
17 +
18 +#include <torch/all.h>
19 +
20 +#include <ATen/cuda/CUDAContext.h>
21 +#include <c10/cuda/CUDAGuard.h>
22 +#include <cuda.h>
23 +#include <cuda_fp16.h>
24 +#include <cuda_runtime.h>
25 +
26 +#include <iostream>
27 +
28 +template <typename T>
29 +inline std::string str(T x) {
30 + return std::to_string(x);
31 +}
32 +
33 +namespace marlin_moe {
34 +
35 +constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
36 +
37 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
38 +
39 +// Instances of `Vec` are used to organize groups of >>registers<<, as needed
40 +// for instance as inputs to tensor core operations. Consequently, all
41 +// corresponding index accesses must be compile-time constants, which is why we
42 +// extensively use `#pragma unroll` throughout the kernel code to guarantee
43 +// this.
44 +template <typename T, int n>
45 +struct Vec {
46 + T elems[n];
47 + __device__ T& operator[](int i) { return elems[i]; }
48 +};
49 +
50 +using I4 = Vec<int, 4>;
51 +
52 +// Matrix fragments for tensor core instructions; their precise layout is
53 +// documented here:
54 +// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
55 +using FragA = Vec<half2, 4>;
56 +using FragB = Vec<half2, 2>;
57 +using FragC = Vec<float, 4>;
58 +using FragS = Vec<half2, 1>; // quantization scales
59 +
60 +// Predicated asynchronous global->shared copy; used for inputs A where we apply
61 +// predication to handle batchsizes that are not multiples of 16.
62 +__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
63 + bool pred = true) {
64 + const int BYTES = 16;
65 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
66 + asm volatile(
67 + "{\n"
68 + " .reg .pred p;\n"
69 + " setp.ne.b32 p, %0, 0;\n"
70 + " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
71 + "}\n" ::"r"((int)pred),
72 + "r"(smem), "l"(glob_ptr), "n"(BYTES));
73 +}
74 +
75 +// Asynchronous global->shared copy
76 +__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
77 + const int BYTES = 16;
78 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
79 + asm volatile(
80 + "{\n"
81 + " cp.async.cg.shared.global [%0], [%1], %2;\n"
82 + "}\n" ::"r"(smem),
83 + "l"(glob_ptr), "n"(BYTES));
84 +}
85 +
86 +// Async copy fence.
87 +__device__ inline void cp_async_fence() {
88 + asm volatile("cp.async.commit_group;\n" ::);
89 +}
90 +
91 +// Wait until at most `n` async copy stages are still pending.
92 +template <int n>
93 +__device__ inline void cp_async_wait() {
94 + asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
95 +}
96 +
97 +// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
98 +// output/accumulation.
99 +__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
100 + FragC& frag_c) {
101 + const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
102 + const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
103 + float* c = reinterpret_cast<float*>(&frag_c);
104 + asm volatile(
105 + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
106 + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
107 + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
108 + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
109 + "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
110 +}
111 +
112 +// Instruction for loading a full 16x16 matrix fragment of operand A from shared
113 +// memory, directly in tensor core layout.
114 +__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
115 + uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
116 + uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
117 + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
118 + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
119 + : "r"(smem));
120 +}
121 +
122 +// Lookup-table based 3-input logical operation; explicitly used for
123 +// dequantization as the compiler does not seem to automatically recognize it in
124 +// all cases.
125 +template <int lut>
126 +__device__ inline int lop3(int a, int b, int c) {
127 + int res;
128 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
129 + : "=r"(res)
130 + : "r"(a), "r"(b), "r"(c), "n"(lut));
131 + return res;
132 +}
133 +
134 +// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
135 +// values. We mostly follow the strategy in the link below, with some small
136 +// changes:
137 +// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
138 +__device__ inline FragB dequant(int q) {
139 + const int LO = 0x000f000f;
140 + const int HI = 0x00f000f0;
141 + const int EX = 0x64006400;
142 + // Guarantee that the `(a & b) | c` operations are LOP3s.
143 + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
144 + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
145 + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
146 + // directly into `SUB` and `ADD`.
147 + const int SUB = 0x64086408;
148 + const int MUL = 0x2c002c00;
149 + const int ADD = 0xd480d480;
150 + FragB frag_b;
151 + frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
152 + *reinterpret_cast<const half2*>(&SUB));
153 + frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
154 + *reinterpret_cast<const half2*>(&MUL),
155 + *reinterpret_cast<const half2*>(&ADD));
156 + return frag_b;
157 +}
158 +
159 +// Multiply dequantized values by the corresponding quantization scale; used
160 +// only for grouped quantization.
161 +__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
162 + half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
163 + frag_b[0] = __hmul2(frag_b[0], s);
164 + frag_b[1] = __hmul2(frag_b[1], s);
165 +}
166 +
167 +// Given 2 floats multiply by 2 scales (halves)
168 +__device__ inline void scale_float(float* c, FragS& s) {
169 + __half* s_ptr = reinterpret_cast<__half*>(&s);
170 + c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
171 + c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
172 +}
173 +
174 +// Same as above, but for act_order (each K is multiplied individually)
175 +__device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
176 + FragS& frag_s_3, FragS& frag_s_4, int i) {
177 + __half2 s_val_1_2;
178 + s_val_1_2.x = reinterpret_cast<__half*>(&frag_s_1)[i];
179 + s_val_1_2.y = reinterpret_cast<__half*>(&frag_s_2)[i];
180 +
181 + __half2 s_val_3_4;
182 + s_val_3_4.x = reinterpret_cast<__half*>(&frag_s_3)[i];
183 + s_val_3_4.y = reinterpret_cast<__half*>(&frag_s_4)[i];
184 +
185 + frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
186 + frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
187 +}
188 +
189 +// Wait until barrier reaches `count`, then lock for current threadblock.
190 +__device__ inline void barrier_acquire(int* lock, int count) {
191 + if (threadIdx.x == 0) {
192 + int state = -1;
193 + do
194 + // Guarantee that subsequent writes by this threadblock will be visible
195 + // globally.
196 + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
197 + : "=r"(state)
198 + : "l"(lock));
199 + while (state != count);
200 + }
201 + __syncthreads();
202 +}
203 +
204 +// Release barrier and increment visitation count.
205 +__device__ inline void barrier_release(int* lock, bool reset = false) {
206 + __syncthreads();
207 + if (threadIdx.x == 0) {
208 + if (reset) {
209 + lock[0] = 0;
210 + return;
211 + }
212 + int val = 1;
213 + // Make sure that all writes since acquiring this barrier are visible
214 + // globally, while releasing the barrier.
215 + asm volatile("fence.acq_rel.gpu;\n");
216 + asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
217 + :
218 + : "l"(lock), "r"(val));
219 + }
220 +}
221 +
222 +// For a given "a" of size [M,K] performs a permutation of the K columns based
223 +// on the given "perm" indices.
224 +__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
225 + int const* __restrict__ perm_int_ptr,
226 + int4* __restrict__ out_int4_ptr, int size_m,
227 + int size_k, int block_rows) {
228 + int start_row = block_rows * blockIdx.x;
229 + int finish_row = start_row + block_rows;
230 + if (finish_row > size_m) {
231 + finish_row = size_m;
232 + }
233 + int cur_block_rows = finish_row - start_row;
234 +
235 + int row_stride = size_k * sizeof(half) / 16;
236 +
237 + auto permute_row = [&](int row) {
238 + int iters = size_k / blockDim.x;
239 + int rest = size_k % blockDim.x;
240 +
241 + int offset = row * row_stride;
242 +
243 + half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
244 + half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
245 +
246 + int base_k = 0;
247 +
248 + for (int i = 0; i < iters; i++) {
249 + int cur_k = base_k + threadIdx.x;
250 + int src_pos = perm_int_ptr[cur_k];
251 +
252 + out_half[cur_k] = a_row_half[src_pos];
253 +
254 + base_k += blockDim.x;
255 + }
256 +
257 + if (rest) {
258 + if (threadIdx.x < rest) {
259 + int cur_k = base_k + threadIdx.x;
260 + int src_pos = perm_int_ptr[cur_k];
261 +
262 + out_half[cur_k] = a_row_half[src_pos];
263 + }
264 + }
265 + };
266 +
267 + for (int i = 0; i < cur_block_rows; i++) {
268 + int cur_row = start_row + i;
269 + if (cur_row < size_m) {
270 + permute_row(cur_row);
271 + }
272 + }
273 +}
274 +
275 +__global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
276 + int* __restrict__ expert_offsets,
277 + int topk_length, int block_size) {
278 + int expert_id = threadIdx.x;
279 + int num_experts = blockDim.x;
280 +
281 + int occurrences = 0;
282 + for (int i = 0; i < topk_length; ++i) {
283 + occurrences += (topk_ids[i] == expert_id);
284 + }
285 + expert_offsets[expert_id + 1] = occurrences;
286 + __syncthreads();
287 +
288 + if (threadIdx.x == 0) {
289 + int tot_offset = 0;
290 + expert_offsets[0] = 0;
291 + for (int i = 0; i < num_experts; ++i) {
292 + tot_offset += ceildiv(expert_offsets[i + 1], block_size) * block_size;
293 + expert_offsets[i + 1] = tot_offset;
294 + }
295 + }
296 + __syncthreads();
297 +}
298 +
299 +template <const int threads, // number of threads in a threadblock
300 + const int thread_m_blocks, // number of 16x16 blocks in the m
301 + // dimension (batchsize) of the
302 + // threadblock
303 + const int thread_n_blocks, // same for n dimension (output)
304 + const int thread_k_blocks, // same for k dimension (reduction)
305 + const int stages, // number of stages for the async global->shared
306 + // fetch pipeline
307 + const bool has_act_order, // whether act_order is enabled
308 + const int group_blocks = -1 // number of consecutive 16x16 blocks
309 + // with a separate quantization scale
310 + >
311 +__device__ inline void MarlinMoESingle(
312 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
313 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
314 + int4* __restrict__ C, // fp16 output buffer of shape mxn
315 + const int* __restrict__ sorted_ids, // int32 sorted ids of experts
316 + const float* __restrict__ topk_weights, // float topk weights
317 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
318 + // (k/groupsize)xn
319 + const int* __restrict__ g_idx, // int32 group indices of shape k
320 + const int* __restrict__ expert_offsets,
321 + int num_groups, // number of scale groups per output channel
322 + int expert_idx, // idx of current expert
323 + int num_experts, // number of experts
324 + int topk, // topk parameter of moe
325 + int prob_m, // batch dimension m
326 + int prob_n, // output dimension n
327 + int prob_k, // reduction dimension k
328 + int tot_m, // total number of rows in A and C
329 + int* locks, // extra global storage for barrier synchronization
330 + bool replicate_input, // do we use the same input for each expert?
331 + bool apply_weights, // apply weights to output
332 + int current_m_block // current m block to start kernel computation from
333 +) {
334 + // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
335 + // better partitioning with less reductions
336 + int parallel = 1;
337 + if (prob_m > 16 * thread_m_blocks) {
338 + parallel = prob_m / (16 * thread_m_blocks);
339 + prob_m = 16 * thread_m_blocks;
340 + }
341 +
342 + int k_tiles = prob_k / 16 / thread_k_blocks;
343 + int n_tiles = prob_n / 16 / thread_n_blocks;
344 + int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
345 +
346 + if constexpr (!has_act_order && group_blocks != -1) {
347 + if (group_blocks >= thread_k_blocks) {
348 + // Ensure that the number of tiles in each stripe is a multiple of the
349 + // groupsize; this avoids an annoying special case where a stripe starts
350 + // in the middle of group.
351 + iters = (group_blocks / thread_k_blocks) *
352 + ceildiv(iters, (group_blocks / thread_k_blocks));
353 + }
354 + }
355 +
356 + int slice_row = (iters * blockIdx.x) % k_tiles;
357 + int slice_col_par = (iters * blockIdx.x) / k_tiles;
358 + int slice_col = slice_col_par;
359 + int slice_iters; // number of threadblock tiles in the current slice
360 + int slice_count =
361 + 0; // total number of active threadblocks in the current slice
362 + int slice_idx; // index of threadblock in current slice; numbered bottom to
363 + // top
364 +
365 + // We can easily implement parallel problem execution by just remapping
366 + // indices and advancing global pointers
367 + if (slice_col_par >= n_tiles) {
368 + locks += (slice_col_par / n_tiles) * n_tiles;
369 + slice_col = slice_col_par % n_tiles;
370 + sorted_ids += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
371 + }
372 +
373 + // Compute all information about the current slice which is required for
374 + // synchronization.
375 + auto init_slice = [&]() {
376 + slice_iters =
377 + iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
378 + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
379 + if (slice_iters == 0) return;
380 + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
381 + slice_count = 1;
382 + slice_idx = 0;
383 + int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
384 + if (col_first <= k_tiles * (slice_col_par + 1)) {
385 + int col_off = col_first - k_tiles * slice_col_par;
386 + slice_count = ceildiv(k_tiles - col_off, iters);
387 + if (col_off > 0) slice_count++;
388 + int delta_first = iters * blockIdx.x - col_first;
389 + if (delta_first < 0 || (col_off == 0 && delta_first == 0))
390 + slice_idx = slice_count - 1;
391 + else {
392 + slice_idx = slice_count - 1 - delta_first / iters;
393 + if (col_off > 0) slice_idx--;
394 + }
395 + }
396 + if (slice_col == n_tiles) {
397 + sorted_ids += 16 * thread_m_blocks;
398 + locks += n_tiles;
399 + slice_col = 0;
400 + }
401 + };
402 + init_slice();
403 +
404 + // A sizes/strides
405 +
406 + // stride of the A matrix in global memory
407 + int a_gl_stride = prob_k / 8;
408 + // stride of an A matrix tile in shared memory
409 + constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
410 + // delta between subsequent A tiles in global memory
411 + constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
412 + // between subsequent accesses within a tile
413 + int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
414 + // between shared memory writes
415 + constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
416 + // between shared memory tile reads
417 + constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
418 + // within a shared memory tile
419 + constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
420 + // overall size of a tile
421 + constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
422 + // number of shared write iterations for a tile
423 + constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
424 +
425 + // B sizes/strides
426 + int b_gl_stride = 16 * prob_n / 32;
427 + constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
428 + int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
429 + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
430 + constexpr int b_sh_wr_delta = threads;
431 + constexpr int b_sh_rd_delta = threads;
432 + constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
433 + constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
434 +
435 + // Scale sizes/strides without act_order
436 + int s_gl_stride = prob_n / 8;
437 + constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
438 + constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
439 + ? thread_k_blocks / group_blocks
440 + : 1;
441 + constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
442 + int s_gl_rd_delta = s_gl_stride;
443 + // Scale size/strides with act_order
444 + constexpr int tb_k = 16 * thread_k_blocks;
445 + constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
446 + // constexpr int act_s_row_stride = 1;
447 + // int act_s_col_stride = act_s_row_stride * num_groups;
448 + int act_s_col_stride = 1;
449 + int act_s_col_warp_stride = act_s_col_stride * 8;
450 + int tb_n_warps = thread_n_blocks / 4;
451 + int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
452 +
453 + constexpr int sorted_sh_stride = threads;
454 + constexpr int sorted_gl_stride = threads;
455 +
456 + // Global A read index of current thread.
457 + int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
458 + (threadIdx.x % a_gl_rd_delta_o);
459 + a_gl_rd += a_gl_rd_delta_o * slice_row;
460 + // Shared write index of current thread.
461 + int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
462 + (threadIdx.x % a_gl_rd_delta_o);
463 + // Shared read index.
464 + int a_sh_rd =
465 + a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
466 + a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
467 +
468 + int b_gl_rd =
469 + b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
470 + b_gl_rd += b_sh_stride * slice_col;
471 + b_gl_rd += b_gl_rd_delta_o * slice_row;
472 + int b_sh_wr = threadIdx.x;
473 + int b_sh_rd = threadIdx.x;
474 +
475 + // For act_order
476 + constexpr int k_iter_size = tb_k / b_sh_wr_iters;
477 + int slice_k_start = tb_k * slice_row;
478 + int slice_k_finish = slice_k_start + tb_k * slice_iters;
479 + int slice_k_start_shared_fetch = slice_k_start;
480 + int slice_n_offset = act_s_col_tb_stride * slice_col;
481 +
482 + // No act_order
483 + int s_gl_rd;
484 + if constexpr (group_blocks == -1 || group_blocks == 0) {
485 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
486 + } else {
487 + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
488 + s_sh_stride * slice_col + threadIdx.x;
489 + }
490 + int s_sh_wr = threadIdx.x;
491 + bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
492 +
493 + // We use a different scale layout for grouped and column-wise quantization as
494 + // we scale a `half2` tile in column-major layout in the former and in
495 + // row-major in the latter case.
496 + int s_sh_rd;
497 + if constexpr (group_blocks != -1)
498 + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
499 + (threadIdx.x % 32) / 4;
500 + else
501 + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
502 + (threadIdx.x % 32) % 4;
503 +
504 + int sh_first_group_id = -1;
505 + int sh_num_groups = -1;
506 + constexpr int sh_max_num_groups = 32;
507 +
508 + int shs_size;
509 + if constexpr (has_act_order)
510 + shs_size = sh_max_num_groups * s_sh_stride + threads;
511 + else
512 + shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
513 +
514 + extern __shared__ int4 sh[];
515 + // Shared memory storage for global fetch pipelines.
516 + int4* sh_a = sh;
517 + int4* sh_b = sh_a + (stages * a_sh_stage);
518 + int4* sh_g_idx = sh_b + (stages * b_sh_stage);
519 + int4* sh_s = sh_g_idx + (stages * g_idx_stage);
520 + int* sh_sorted = (int*)(sh_s + shs_size);
521 +
522 + // Precompute which thread should not read memory in which iterations; this is
523 + // needed if there are more threads than required for a certain tilesize or
524 + // when the batchsize is not a multiple of 16.
525 + bool a_sh_wr_pred[a_sh_wr_iters];
526 + #pragma unroll
527 + for (int i = 0; i < a_sh_wr_iters; i++) {
528 + int a_idx = a_sh_wr_delta * i + a_sh_wr;
529 + int row = a_idx / a_gl_rd_delta_o;
530 + if (row >= prob_m) {
531 + a_sh_wr_pred[i] = false;
532 + } else {
533 + a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
534 + }
535 + }
536 +
537 + // To ensure that writing and reading A tiles to/from shared memory, the
538 + // latter in fragment format, is fully bank conflict free, we need to use a
539 + // rather fancy XOR-based layout. The key here is that neither reads nor
540 + // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
541 + // same shared memory banks. Further, it seems (based on NSight-Compute) that
542 + // each warp must also write a consecutive memory segment?
543 + auto transform_a = [&](int i) {
544 + int row = i / a_gl_rd_delta_o;
545 + return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
546 + };
547 + // Since the computation of this remapping is non-trivial and, due to our main
548 + // loop unrolls, all shared memory accesses are static, we simply precompute
549 + // both transformed reads and writes.
550 + int a_sh_wr_trans[a_sh_wr_iters];
551 + #pragma unroll
552 + for (int i = 0; i < a_sh_wr_iters; i++)
553 + a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
554 + int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
555 + #pragma unroll
556 + for (int i = 0; i < b_sh_wr_iters; i++) {
557 + #pragma unroll
558 + for (int j = 0; j < thread_m_blocks; j++)
559 + a_sh_rd_trans[i][j] =
560 + transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
561 + }
562 +
563 + // Since B-accesses have non-constant stride they have to be computed at
564 + // runtime; we break dependencies between subsequent accesses with a tile by
565 + // maintining multiple pointers (we have enough registers), a tiny
566 + // optimization.
567 + const int4* B_ptr[b_sh_wr_iters];
568 + #pragma unroll
569 + for (int i = 0; i < b_sh_wr_iters; i++)
570 + B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
571 +
572 + // Register storage for double buffer of shared memory reads.
573 + FragA frag_a[2][thread_m_blocks];
574 + I4 frag_b_quant[2];
575 + FragC frag_c[thread_m_blocks][4][2];
576 + FragS frag_s[2][4]; // No act-order
577 + FragS act_frag_s[2][4][4]; // For act-order
578 +
579 + // Zero accumulators.
580 + auto zero_accums = [&]() {
581 + #pragma unroll
582 + for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
583 + reinterpret_cast<float*>(frag_c)[i] = 0;
584 + };
585 +
586 + auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
587 + int last_group_id) {
588 + sh_first_group_id = first_group_id;
589 + sh_num_groups = last_group_id - first_group_id + 1;
590 +
591 + if (sh_num_groups < sh_max_num_groups) {
592 + sh_num_groups = sh_max_num_groups;
593 + }
594 +
595 + if (sh_first_group_id + sh_num_groups > num_groups) {
596 + sh_num_groups = num_groups - sh_first_group_id;
597 + }
598 +
599 + int row_offset = first_group_id * s_gl_stride;
600 +
601 + if (is_async) {
602 + for (int i = 0; i < sh_num_groups; i++) {
603 + if (threadIdx.x < s_sh_stride) {
604 + cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
605 + &scales_ptr[row_offset + (i * s_gl_stride) +
606 + slice_n_offset + threadIdx.x]);
607 + }
608 + }
609 + } else {
610 + for (int i = 0; i < sh_num_groups; i++) {
611 + if (threadIdx.x < s_sh_stride) {
612 + sh_s[(i * s_sh_stride) + threadIdx.x] =
613 + scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
614 + threadIdx.x];
615 + }
616 + }
617 + }
618 + };
619 + // Asynchronously fetch the next A, B and s tile from global to the next
620 + // shared memory pipeline location.
621 + auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
622 + if (pred) {
623 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
624 + #pragma unroll
625 + for (int i = 0; i < a_sh_wr_iters; i++) {
626 + int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off;
627 + int row = a_idx / a_gl_stride;
628 + int sorted_row =
629 + replicate_input ? sorted_ids[row] / topk : sorted_ids[row];
630 + int new_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride;
631 + if (sorted_row < tot_m * (replicate_input ? 1 : topk) &&
632 + new_idx < a_gl_stride * tot_m * (replicate_input ? 1 : topk)) {
633 + cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[new_idx],
634 + a_sh_wr_pred[i]);
635 + }
636 + }
637 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
638 + #pragma unroll
639 + for (int i = 0; i < b_sh_wr_iters; i++) {
640 + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
641 + B_ptr[i] += b_gl_rd_delta_o;
642 + }
643 +
644 + if constexpr (has_act_order) {
645 + // Fetch g_idx thread-block portion
646 + int full_pipe = a_off;
647 + int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
648 + if (cur_k < prob_k && cur_k < slice_k_finish) {
649 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
650 +
651 + int4 const* cur_g_idx_stage_ptr =
652 + reinterpret_cast<int4 const*>(&g_idx[cur_k]);
653 +
654 + if (threadIdx.x < g_idx_stage) {
655 + cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
656 + &cur_g_idx_stage_ptr[threadIdx.x]);
657 + }
658 + }
659 + } else {
660 + if constexpr (group_blocks != -1) {
661 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
662 +
663 + if constexpr (group_blocks >= thread_k_blocks) {
664 + // Only fetch scales if this tile starts a new group
665 + if (pipe % (group_blocks / thread_k_blocks) == 0) {
666 + if (s_sh_wr_pred) {
667 + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
668 + }
669 + s_gl_rd += s_gl_rd_delta;
670 + }
671 + } else {
672 + for (int i = 0; i < s_tb_groups; i++) {
673 + if (s_sh_wr_pred) {
674 + cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
675 + &scales_ptr[s_gl_rd]);
676 + }
677 + s_gl_rd += s_gl_rd_delta;
678 + }
679 + }
680 + }
681 + }
682 + }
683 + // Insert a fence even when we are winding down the pipeline to ensure that
684 + // waiting is also correct at this point.
685 + cp_async_fence();
686 + };
687 +
688 + // TODO we are currently hitting illegal memory accesses when fetching
689 + // sorted_ids to shared data: fix this
690 + auto fetch_sorted_ids_to_shared = [&]() {
691 + const int mpt = ceildiv(prob_m, threads);
692 + for (int i = 0; i < mpt; i++) {
693 + if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
694 + sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
695 + sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
696 + }
697 + }
698 + };
699 +
700 + // Wait until the next thread tile has been loaded to shared memory.
701 + auto wait_for_stage = [&]() {
702 + // We only have `stages - 2` active fetches since we are double buffering
703 + // and can only issue the next fetch when it is guaranteed that the previous
704 + // shared memory load is fully complete (as it may otherwise be
705 + // overwritten).
706 + cp_async_wait<stages - 2>();
707 + __syncthreads();
708 + };
709 +
710 + // Load the next sub-tile from the current location in the shared memory pipe
711 + // into the current register buffer.
712 + auto fetch_to_registers = [&](int k, int pipe) {
713 + int4* sh_a_stage = sh_a + a_sh_stage * pipe;
714 + #pragma unroll
715 + for (int i = 0; i < thread_m_blocks; i++)
716 + ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
717 + int4* sh_b_stage = sh_b + b_sh_stage * pipe;
718 + frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
719 + &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
720 + };
721 +
722 + bool is_same_group[stages];
723 + int same_group_id[stages];
724 +
725 + auto init_same_group = [&](int pipe) {
726 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
727 + int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
728 +
729 + int group_id_1 = sh_g_idx_int_ptr[0];
730 + int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
731 +
732 + is_same_group[pipe] = group_id_1 == group_id_2;
733 + same_group_id[pipe] = group_id_1;
734 + };
735 +
736 + auto fetch_scales_to_registers = [&](int k, int full_pipe) {
737 + int pipe = full_pipe % stages;
738 +
739 + if constexpr (!has_act_order) {
740 + // No act-order case
741 + if constexpr (group_blocks != -1) {
742 + if constexpr (group_blocks >= thread_k_blocks) {
743 + int4* sh_s_stage =
744 + sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
745 + (pipe / (group_blocks / thread_k_blocks)));
746 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
747 + } else {
748 + int warp_id = threadIdx.x / 32;
749 + int n_warps = thread_n_blocks / 4;
750 +
751 + int warp_row = warp_id / n_warps;
752 +
753 + int cur_k = warp_row * 16;
754 + cur_k += k_iter_size * (k % b_sh_wr_iters);
755 +
756 + int k_blocks = cur_k / 16;
757 + int cur_group_id = k_blocks / group_blocks;
758 +
759 + int4* sh_s_stage = sh_s + s_sh_stage * pipe;
760 +
761 + reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
762 + sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
763 + }
764 + }
765 +
766 + return;
767 + }
768 +
769 + // Act-order case
770 +
771 + // Determine K of the "current" thread-block
772 + int cur_k = slice_k_start + tb_k * full_pipe;
773 + if (cur_k >= prob_k || cur_k >= slice_k_finish) {
774 + return;
775 + }
776 +
777 + // Reset (to current thread-block) since we read g_idx portion from the
778 + // shared memory
779 + cur_k = 0;
780 +
781 + // Progress to current iteration
782 + cur_k += k_iter_size * (k % b_sh_wr_iters);
783 +
784 + // Determine "position" inside the thread-block (based on warp and
785 + // thread-id)
786 + int warp_id = threadIdx.x / 32;
787 + int n_warps =
788 + thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
789 +
790 + int warp_row = warp_id / n_warps;
791 + int warp_col = warp_id % n_warps;
792 +
793 + cur_k += warp_row * 16;
794 +
795 + int th_id = threadIdx.x % 32;
796 + cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
797 +
798 + int s_col_shift =
799 + /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
800 + (th_id / 4) * act_s_col_stride;
801 +
802 + if (is_same_group[pipe]) {
803 + if (k % 2 == 0) {
804 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
805 + sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
806 + s_col_shift];
807 + } else {
808 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
809 + *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
810 + }
811 +
812 + for (int i = 1; i < 4; i++) {
813 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
814 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
815 + }
816 + return;
817 + }
818 +
819 + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
820 + int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
821 +
822 + constexpr int k_frag_offsets[4] = {0, 1, 8,
823 + 9}; // Tensor core offsets per thread
824 +
825 + #pragma unroll
826 + for (int i = 0; i < 4; i++) {
827 + int actual_k = cur_k + k_frag_offsets[i];
828 +
829 + int group_id = sh_g_idx_int_ptr[actual_k];
830 + int rel_group_id = group_id - sh_first_group_id;
831 +
832 + *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
833 + sh_s[rel_group_id * s_sh_stride + s_col_shift];
834 + }
835 + };
836 +
837 + // Execute the actual tensor core matmul of a sub-tile.
838 + auto matmul = [&](int k) {
839 + // We have the m dimension as the inner loop in order to encourage overlapping
840 + // dequantization and matmul operations.
841 + #pragma unroll
842 + for (int j = 0; j < 4; j++) {
843 + int b_quant = frag_b_quant[k % 2][j];
844 + int b_quant_shift = b_quant >> 8;
845 +
846 + FragB frag_b0 = dequant(b_quant);
847 +
848 + // Apply scale to frag_b0
849 + if constexpr (has_act_order) {
850 + scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
851 + act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
852 + } else {
853 + if constexpr (group_blocks != -1) {
854 + scale(frag_b0, frag_s[k % 2][j], 0);
855 + }
856 + }
857 +
858 + FragB frag_b1 = dequant(b_quant_shift);
859 +
860 + // Apply scale to frag_b1
861 + if constexpr (has_act_order) {
862 + scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
863 + act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
864 +
865 + } else {
866 + if constexpr (group_blocks != -1) {
867 + scale(frag_b1, frag_s[k % 2][j], 1);
868 + }
869 + }
870 +
871 + #pragma unroll
872 + for (int i = 0; i < thread_m_blocks; i++) {
873 + mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
874 + mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
875 + }
876 + }
877 + };
878 +
879 + // Since we slice across the k dimension of a tile in order to increase the
880 + // number of warps while keeping the n dimension of a tile reasonable, we have
881 + // multiple warps that accumulate their partial sums of the same output
882 + // location; which we have to reduce over in the end. We do in shared memory.
883 + auto thread_block_reduce = [&]() {
884 + constexpr int red_off = threads / b_sh_stride / 2;
885 + if (red_off >= 1) {
886 + int red_idx = threadIdx.x / b_sh_stride;
887 + constexpr int red_sh_stride = b_sh_stride * 4 * 2;
888 + constexpr int red_sh_delta = b_sh_stride;
889 + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
890 + (threadIdx.x % b_sh_stride);
891 +
892 + // Parallel logarithmic shared memory reduction. We make sure to avoid any
893 + // unnecessary read or write iterations, e.g., for two warps we write only
894 + // once by warp 1 and read only once by warp 0.
895 +
896 + #pragma unroll
897 + for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
898 + #pragma unroll
899 + for (int i = red_off; i > 0; i /= 2) {
900 + if (i <= red_idx && red_idx < 2 * i) {
901 + #pragma unroll
902 + for (int j = 0; j < 4 * 2; j++) {
903 + int red_sh_wr =
904 + red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
905 + if (i < red_off) {
906 + float* c_rd =
907 + reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
908 + float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
909 + #pragma unroll
910 + for (int k = 0; k < 4; k++)
911 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
912 + c_rd[k] + c_wr[k];
913 + }
914 + sh[red_sh_wr] =
915 + reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
916 + }
917 + }
918 + __syncthreads();
919 + }
920 + if (red_idx == 0) {
921 + #pragma unroll
922 + for (int i = 0; i < 4 * 2; i++) {
923 + float* c_rd =
924 + reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
925 + #pragma unroll
926 + for (int j = 0; j < 4; j++)
927 + reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
928 + c_rd[j];
929 + }
930 + }
931 + __syncthreads();
932 + }
933 + }
934 + };
935 +
936 + // Since multiple threadblocks may process parts of the same column slice, we
937 + // finally have to globally reduce over the results. As the striped
938 + // partitioning minimizes the number of such reductions and our outputs are
939 + // usually rather small, we perform this reduction serially in L2 cache.
940 + auto global_reduce = [&](bool first = false, bool last = false) {
941 + // We are very careful here to reduce directly in the output buffer to
942 + // maximize L2 cache utilization in this step. To do this, we write out
943 + // results in FP16 (but still reduce with FP32 compute).
944 + constexpr int active_threads = 32 * thread_n_blocks / 4;
945 + if (threadIdx.x < active_threads) {
946 + int c_gl_stride = prob_n / 8;
947 + int c_gl_wr_delta_o = 8 * c_gl_stride;
948 + int c_gl_wr_delta_i = 4 * (active_threads / 32);
949 + int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
950 + 4 * (threadIdx.x / 32) + threadIdx.x % 4;
951 + c_gl_wr += (2 * thread_n_blocks) * slice_col;
952 + constexpr int c_sh_wr_delta = active_threads;
953 + int c_sh_wr = threadIdx.x;
954 +
955 + int row = (threadIdx.x % 32) / 4;
956 +
957 + if (!first) {
958 + // Interestingly, doing direct global accesses here really seems to mess up
959 + // the compiler and lead to slowdowns, hence we also use async-copies even
960 + // though these fetches are not actually asynchronous.
961 + #pragma unroll
962 + for (int i = 0; i < thread_m_blocks * 4; i++) {
963 + int c_idx =
964 + c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
965 + int sorted_row = sorted_ids[c_idx / c_gl_stride];
966 + int new_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride;
967 + cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i], &C[new_idx],
968 + sorted_row < tot_m * topk &&
969 + (8 * (i / 2) + row < prob_m &&
970 + (i < (thread_m_blocks - 1) * 4 ||
971 + sorted_ids[8 * (i / 2) + row] < tot_m * topk)));
972 + }
973 + cp_async_fence();
974 + cp_async_wait<0>();
975 + }
976 +
977 + #pragma unroll
978 + for (int i = 0; i < thread_m_blocks * 4; i++) {
979 + if (8 * (i / 2) + row < prob_m &&
980 + (i < (thread_m_blocks - 1) * 4 ||
981 + sorted_ids[8 * (i / 2) + row] < tot_m * topk)) {
982 + if (!first) {
983 + int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
984 + #pragma unroll
985 + for (int j = 0; j < 2 * 4; j++) {
986 + reinterpret_cast<float*>(
987 + &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
988 + __half2float(reinterpret_cast<__half*>(&c_red)[j]);
989 + }
990 + }
991 + if (!last) {
992 + int4 c;
993 + #pragma unroll
994 + for (int j = 0; j < 2 * 4; j++) {
995 + reinterpret_cast<__half*>(&c)[j] =
996 + __float2half(reinterpret_cast<float*>(
997 + &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
998 + }
999 + int c_idx =
1000 + c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
1001 + int row = sorted_ids[c_idx / c_gl_stride];
1002 + if (row < tot_m * topk) {
1003 + int new_idx = row * c_gl_stride + c_idx % c_gl_stride;
1004 + C[new_idx] = c;
1005 + }
1006 + }
1007 + }
1008 + }
1009 + }
1010 + };
1011 +
1012 + // Write out the reduce final result in the correct layout. We only actually
1013 + // reshuffle matrix fragments in this step, the reduction above is performed
1014 + // in fragment layout.
1015 + auto write_result = [&]() {
1016 + int c_gl_stride = prob_n / 8;
1017 + constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
1018 + int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
1019 + constexpr int c_sh_rd_delta =
1020 + c_sh_stride * (threads / (2 * thread_n_blocks));
1021 +
1022 + int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1023 + (threadIdx.x % (2 * thread_n_blocks));
1024 + c_gl_wr += (2 * thread_n_blocks) * slice_col;
1025 + int c_sh_wr =
1026 + (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
1027 + c_sh_wr += 32 * (threadIdx.x / 32);
1028 + int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1029 + (threadIdx.x % (2 * thread_n_blocks));
1030 +
1031 + int c_gl_wr_end = c_gl_stride * prob_m;
1032 +
1033 + // We first reorder in shared memory to guarantee the most efficient final
1034 + // global write patterns
1035 + auto write = [&](int idx, float c0, float c1, FragS& s) {
1036 + half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1037 +
1038 + // For per-column quantization we finally apply the scale here
1039 + if constexpr (!has_act_order && group_blocks == -1) {
1040 + res = __hmul2(res, s[0]);
1041 + }
1042 +
1043 + ((half2*)sh)[idx] = res;
1044 + };
1045 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1046 + #pragma unroll
1047 + for (int i = 0; i < thread_m_blocks; i++) {
1048 + #pragma unroll
1049 + for (int j = 0; j < 4; j++) {
1050 + int wr = c_sh_wr + 8 * j;
1051 + write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
1052 + frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
1053 + write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
1054 + frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
1055 + write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
1056 + frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
1057 + write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
1058 + frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
1059 + }
1060 + c_sh_wr += 16 * (4 * c_sh_stride);
1061 + }
1062 + }
1063 + __syncthreads();
1064 +
1065 + #pragma unroll
1066 + for (int i = 0;
1067 + i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1068 + i++) {
1069 + if (c_gl_wr < c_gl_wr_end) {
1070 + int row = sorted_ids[c_gl_wr / c_gl_stride];
1071 + if (row < tot_m * topk) {
1072 + int off = row * c_gl_stride + c_gl_wr % c_gl_stride;
1073 + if (!apply_weights) {
1074 + C[off] = sh[c_sh_rd];
1075 + } else {
1076 + __half* ctrg = reinterpret_cast<__half*>(&C[off]);
1077 + __half* csrc = reinterpret_cast<__half*>(&sh[c_sh_rd]);
1078 + for (int j = 0; j < 8; ++j) {
1079 + ctrg[j] = __float2half(topk_weights[row] * __half2float(csrc[j]));
1080 + }
1081 + }
1082 + c_gl_wr += c_gl_wr_delta;
1083 + c_sh_rd += c_sh_rd_delta;
1084 + }
1085 + }
1086 + }
1087 + };
1088 +
1089 + // Start global fetch and register load pipelines.
1090 + auto start_pipes = [&]() {
1091 + // TODO re-enable after fixing this function
1092 + // fetch_sorted_ids_to_shared();
1093 + __syncthreads();
1094 +
1095 + #pragma unroll
1096 + for (int i = 0; i < stages - 1; i++) {
1097 + if (has_act_order && i == 0) {
1098 + int last_g_idx = slice_k_start + stages * tb_k * 2;
1099 + if (last_g_idx >= prob_k) {
1100 + last_g_idx = prob_k - 1;
1101 + }
1102 + fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1103 + }
1104 + fetch_to_shared(i, i, i < slice_iters);
1105 + }
1106 +
1107 + zero_accums();
1108 + wait_for_stage();
1109 + init_same_group(0);
1110 + fetch_to_registers(0, 0);
1111 + fetch_scales_to_registers(0, 0);
1112 + a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1113 + slice_k_start_shared_fetch += tb_k * (stages - 1);
1114 + };
1115 + if (slice_iters) {
1116 + start_pipes();
1117 + }
1118 +
1119 + // Main loop.
1120 + while (slice_iters) {
1121 + // We unroll over both the global fetch and the register load pipeline to
1122 + // ensure all shared memory accesses are static. Note that both pipelines
1123 + // have even length meaning that the next iteration will always start at
1124 + // index 0.
1125 + #pragma unroll
1126 + for (int pipe = 0; pipe < stages;) {
1127 + #pragma unroll
1128 + for (int k = 0; k < b_sh_wr_iters; k++) {
1129 + fetch_to_registers(k + 1, pipe % stages);
1130 + fetch_scales_to_registers(k + 1, pipe);
1131 + if (k == b_sh_wr_iters - 2) {
1132 + fetch_to_shared((pipe + stages - 1) % stages, pipe,
1133 + slice_iters >= stages);
1134 + pipe++;
1135 + wait_for_stage();
1136 + init_same_group(pipe % stages);
1137 + }
1138 + matmul(k);
1139 + }
1140 + slice_iters--;
1141 + if (slice_iters == 0) {
1142 + break;
1143 + }
1144 + }
1145 +
1146 + a_gl_rd += a_gl_rd_delta_o * stages;
1147 + slice_k_start += tb_k * stages;
1148 + slice_k_start_shared_fetch += tb_k * stages;
1149 +
1150 + if constexpr (has_act_order) {
1151 + int first_group_id = g_idx[slice_k_start];
1152 + int last_g_idx = slice_k_start + stages * tb_k * 2;
1153 + if (last_g_idx >= prob_k) {
1154 + last_g_idx = prob_k - 1;
1155 + }
1156 + int last_group_id = g_idx[last_g_idx];
1157 + if (last_group_id >= sh_first_group_id + sh_num_groups) {
1158 + fetch_scales_to_shared(false, first_group_id, last_group_id);
1159 + __syncthreads();
1160 + }
1161 + }
1162 +
1163 + // Process results and, if necessary, proceed to the next column slice.
1164 + // While this pattern may not be the most readable, other ways of writing
1165 + // the loop seemed to noticeably worse performance after compilation.
1166 + if (slice_iters == 0) {
1167 + cp_async_wait<0>();
1168 + bool last = slice_idx == slice_count - 1;
1169 + // For per-column scales, we only fetch them here in the final step before
1170 + // write-out
1171 + if constexpr (!has_act_order && group_blocks == -1) {
1172 + if (last) {
1173 + if (s_sh_wr_pred) {
1174 + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1175 + }
1176 + cp_async_fence();
1177 + }
1178 + }
1179 +
1180 + thread_block_reduce();
1181 + if constexpr (!has_act_order && group_blocks == -1) {
1182 + if (last) {
1183 + cp_async_wait<0>();
1184 + __syncthreads();
1185 + if (threadIdx.x / 32 < thread_n_blocks / 4) {
1186 + reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1187 + reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1188 + }
1189 + }
1190 + }
1191 + if (slice_count > 1) { // only globally reduce if there is more than one
1192 + // block in a slice
1193 + barrier_acquire(&locks[slice_col], slice_idx);
1194 + global_reduce(slice_idx == 0, last);
1195 + barrier_release(&locks[slice_col], last);
1196 + }
1197 + if (last) // only the last block in a slice actually writes the result
1198 + write_result();
1199 + slice_row = 0;
1200 + slice_col_par++;
1201 + slice_col++;
1202 + init_slice();
1203 + if (slice_iters) {
1204 + a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1205 + (threadIdx.x % a_gl_rd_delta_o);
1206 + #pragma unroll
1207 + for (int i = 0; i < b_sh_wr_iters; i++)
1208 + B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1209 + if (slice_col == 0) {
1210 + #pragma unroll
1211 + for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1212 + }
1213 +
1214 + // Update slice k/n for scales loading
1215 + if constexpr (has_act_order) {
1216 + slice_k_start = tb_k * slice_row;
1217 + slice_k_finish = slice_k_start + tb_k * slice_iters;
1218 + slice_k_start_shared_fetch = slice_k_start;
1219 + slice_n_offset = act_s_col_tb_stride * slice_col;
1220 +
1221 + } else {
1222 + s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1223 + }
1224 + start_pipes();
1225 + }
1226 + }
1227 + }
1228 +}
1229 +
1230 +template <const int threads, // number of threads in a threadblock
1231 + const int thread_m_blocks, // number of 16x16 blocks in the m
1232 + // dimension (batchsize) of the
1233 + // threadblock
1234 + const int thread_n_blocks, // same for n dimension (output)
1235 + const int thread_k_blocks, // same for k dimension (reduction)
1236 + const int stages, // number of stages for the async global->shared
1237 + // fetch pipeline
1238 + const bool has_act_order, // whether act_order is enabled
1239 + const int group_blocks = -1 // number of consecutive 16x16 blocks
1240 + // with a separate quantization scale
1241 + >
1242 +__global__ void MarlinMoE(
1243 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
1244 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1245 + int4* __restrict__ C, // fp16 output buffer of shape mxn
1246 + const int* __restrict__ sorted_ids_base, // int32 sorted ids of experts
1247 + const float* __restrict__ topk_weights, // float topk weights
1248 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1249 + // (k/groupsize)xn
1250 + const int* __restrict__ g_idx, // int32 group indices of shape k
1251 + const int* __restrict__ expert_offsets,
1252 + int num_groups, // number of scale groups per output channel
1253 + int expert_idx, // idx of current expert
1254 + int num_experts, // number of experts
1255 + int topk, // topk parameter of moe
1256 + int prob_m, // batch dimension m
1257 + int prob_n, // output dimension n
1258 + int prob_k, // reduction dimension k
1259 + int tot_m, // total number of rows in A and C
1260 + int* locks, // extra global storage for barrier synchronization
1261 + bool replicate_input, // do we use the same input for each expert?
1262 + bool apply_weights, // apply weights to output
1263 + int current_m_block, // current m block to start kernel computation from
1264 + int max_par // maximum parallelism
1265 +) {
1266 + int m_block_ctr = current_m_block;
1267 +
1268 + const int* sorted_ids_expert =
1269 + sorted_ids_base + expert_offsets[expert_idx] + m_block_ctr * 4 * max_par;
1270 + int tot_its = expert_offsets[expert_idx + 1] - expert_offsets[expert_idx];
1271 + if (tot_its == 0) {
1272 + return;
1273 + }
1274 + int tot_m_blocks = ceildiv(tot_its, 16);
1275 + int pad = 16 * tot_m_blocks - tot_its;
1276 +
1277 + if (m_block_ctr >= tot_m_blocks) {
1278 + return;
1279 + }
1280 +
1281 + int max_block = tot_m_blocks - m_block_ctr;
1282 + prob_m = tot_its - 16 * m_block_ctr;
1283 +
1284 + int par = 1;
1285 + if (max_block > 4) {
1286 + // Note that parallel > 1 currently only works for inputs without any
1287 + // padding
1288 + par = (16 * max_block - pad) / 64;
1289 + par = min((16 * max_block - pad) / 64, max_par);
1290 + prob_m = 64 * par;
1291 + m_block_ctr += 4 * (par - 1);
1292 + max_block = 4;
1293 + }
1294 +
1295 + if (max_block == 1) {
1296 + MarlinMoESingle<threads, 1, thread_n_blocks, thread_k_blocks, stages,
1297 + has_act_order, group_blocks>(
1298 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1299 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1300 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1301 + current_m_block);
1302 + } else if (max_block == 2) {
1303 + MarlinMoESingle<threads, 2, thread_n_blocks, thread_k_blocks, stages,
1304 + has_act_order, group_blocks>(
1305 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1306 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1307 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1308 + current_m_block);
1309 + } else if (max_block == 3) {
1310 + MarlinMoESingle<threads, 3, thread_n_blocks, thread_k_blocks, stages,
1311 + has_act_order, group_blocks>(
1312 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1313 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1314 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1315 + current_m_block);
1316 + } else {
1317 + MarlinMoESingle<threads, 4, thread_n_blocks, thread_k_blocks, stages,
1318 + has_act_order, group_blocks>(
1319 + A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1320 + expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1321 + prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1322 + current_m_block);
1323 + }
1324 +}
1325 +
1326 +#else
1327 +
1328 +__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
1329 + int const* __restrict__ perm_int_ptr,
1330 + int4* __restrict__ out_int4_ptr, int size_m,
1331 + int size_k, int block_rows) {
1332 + // Marlin is not implemented yet for SM < 8.0
1333 + assert(false);
1334 + return;
1335 +}
1336 +
1337 +__global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
1338 + int* __restrict__ expert_offsets,
1339 + int topk_length, int block_size) {
1340 + // Marlin is not implemented yet for SM < 8.0
1341 + assert(false);
1342 + return;
1343 +}
1344 +
1345 +template <const int threads, // number of threads in a threadblock
1346 + const int thread_m_blocks, // number of 16x16 blocks in the m
1347 + // dimension (batchsize) of the
1348 + // threadblock
1349 + const int thread_n_blocks, // same for n dimension (output)
1350 + const int thread_k_blocks, // same for k dimension (reduction)
1351 + const int stages, // number of stages for the async global->shared
1352 + // fetch pipeline
1353 + const bool has_act_order, // whether act_order is enabled
1354 + const int group_blocks = -1 // number of consecutive 16x16 blocks
1355 + // with a separate quantization scale
1356 + >
1357 +__global__ void MarlinMoE(
1358 + const int4* __restrict__ A, // fp16 input matrix of shape mxk
1359 + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1360 + int4* __restrict__ C, // fp16 output buffer of shape mxn
1361 + const int* __restrict__ sorted_ids, // int32 sorted ids of experts
1362 + const float* __restrict__ topk_weights, // float topk weights
1363 + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1364 + // (k/groupsize)xn
1365 + const int* __restrict__ g_idx, // int32 group indices of shape k
1366 + const int* __restrict__ expert_offsets,
1367 + int num_groups, // number of scale groups per output channel
1368 + int expert_idx, // idx of current expert
1369 + int num_experts, // number of experts
1370 + int topk, // topk parameter of moe
1371 + int prob_m, // batch dimension m
1372 + int prob_n, // output dimension n
1373 + int prob_k, // reduction dimension k
1374 + int tot_m, // total number of rows in A and C
1375 + int* locks, // extra global storage for barrier synchronization
1376 + bool replicate_input, // do we use the same input for each expert?
1377 + bool apply_weights, // apply weights to output
1378 + int current_m_block, // current m block to start kernel computation from
1379 + int max_par // maximum parallelism
1380 +) {
1381 + // Marlin is not implemented yet for SM < 8.0
1382 + assert(false);
1383 + return;
1384 +}
1385 +
1386 +#endif
1387 +
1388 +// 8 warps are a good choice since every SM has 4 schedulers and having more
1389 +// than 1 warp per schedule allows some more latency hiding. At the same time,
1390 +// we want relatively few warps to have many registers per warp and small tiles.
1391 +const int USER_THREADS =
1392 + 256; // Note: This is only used with user-provided thread_k/n
1393 +const int STAGES = 4; // 4 pipeline stages fit into shared memory
1394 +// const int SHARED_MEM =
1395 +// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1396 +
1397 +static constexpr int min_thread_n = 64;
1398 +static constexpr int min_thread_k = 64;
1399 +
1400 +#define __CALL_IF_MOE(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1401 + HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1402 + else if (thread_m_blocks == THREAD_M_BLOCKS && \
1403 + thread_n_blocks == THREAD_N_BLOCKS && \
1404 + thread_k_blocks == THREAD_K_BLOCKS && \
1405 + has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1406 + num_threads == NUM_THREADS) { \
1407 + cudaFuncSetAttribute( \
1408 + MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1409 + THREAD_K_BLOCKS, STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1410 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1411 + MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1412 + STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1413 + <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1414 + A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1415 + g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1416 + num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1417 + replicate_input, apply_weights, m_block, max_par); \
1418 + }
1419 +
1420 +typedef struct {
1421 + int thread_k;
1422 + int thread_n;
1423 + int num_threads;
1424 +} thread_config_t;
1425 +
1426 +thread_config_t small_batch_thread_configs[] = {
1427 + // Ordered by priority
1428 +
1429 + // thread_k, thread_n, num_threads
1430 + {128, 128, 256}, // Default
1431 + {128, 64, 128}, // Reduce N 2X, same K
1432 + {64, 256, 256}, // Reduce K 2X, increase N 2X
1433 + {64, 128, 128}, // Reduce K 2X, same N
1434 +};
1435 +
1436 +thread_config_t large_batch_thread_configs[] = {
1437 + // Ordered by priority
1438 +
1439 + // thread_k, thread_n, num_threads
1440 + {64, 256, 256}, // Default
1441 + {128, 128, 256}, // Reduce N 2X, increase K 2X
1442 + {64, 128, 128}, // Reduce N 2X, same K
1443 + {128, 64, 128}, // Reduce N 4X, increase K 2X
1444 +};
1445 +
1446 +bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
1447 + int prob_k) {
1448 + // Sanity
1449 + if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1450 + th_config.num_threads == -1) {
1451 + return false;
1452 + }
1453 +
1454 + // Verify K/N are divisible by thread K/N
1455 + if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) {
1456 + return false;
1457 + }
1458 +
1459 + // thread_k can be only 128 or 64 (because it must be less than groupsize
1460 + // which is 128)
1461 + if (th_config.thread_k != 128 && th_config.thread_k != 64) {
1462 + return false;
1463 + }
1464 +
1465 + // Verify min for thread K/N
1466 + if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) {
1467 + return false;
1468 + }
1469 +
1470 + // num_threads must be at least 128 (= 4 warps)
1471 + if (th_config.num_threads < 128) {
1472 + return false;
1473 + }
1474 +
1475 + return true;
1476 +}
1477 +
1478 +thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1479 + if (prob_m <= 16) {
1480 + for (auto th_config : small_batch_thread_configs) {
1481 + if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1482 + return th_config;
1483 + }
1484 + }
1485 +
1486 + } else {
1487 + for (auto th_config : large_batch_thread_configs) {
1488 + if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1489 + return th_config;
1490 + }
1491 + }
1492 + }
1493 +
1494 + return thread_config_t{-1, -1, -1};
1495 +}
1496 +
1497 +#define CALL_IF_MOE(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1498 + __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1499 + __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1500 + __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1501 + __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1502 + \
1503 + __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1504 + __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1505 + __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1506 + __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1507 + \
1508 + __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1509 + __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1510 + __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1511 + __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1512 + \
1513 + __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1514 + __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1515 + __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1516 + __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1517 + \
1518 + __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1519 + __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1520 + __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1521 + __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1522 +
1523 +void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1524 + const void* sorted_ids, const void* topk_weights,
1525 + const void* topk_ids, const void* s, const void* g_idx,
1526 + const void* perm, void* a_tmp, void* expert_offsets,
1527 + int prob_m, int prob_n, int prob_k, void* workspace,
1528 + bool has_act_order, bool is_k_full, int num_groups,
1529 + int group_size, int num_experts, int topk,
1530 + int moe_block_size, int dev, cudaStream_t stream,
1531 + int thread_k, int thread_n, int sms, int max_par,
1532 + bool replicate_input, bool apply_weights) {
1533 + TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1534 + ", ", prob_n, ", ", prob_k, "]");
1535 +
1536 + if (sms == -1) {
1537 + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1538 + }
1539 +
1540 + // Set thread config
1541 + thread_config_t th_config;
1542 + if (thread_k != -1 && thread_n != -1) {
1543 + // User-defined config
1544 + th_config = thread_config_t{thread_k, thread_n, USER_THREADS};
1545 + } else {
1546 + // Auto config
1547 + th_config = determine_thread_config(prob_m, prob_n, prob_k);
1548 + }
1549 +
1550 + TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1551 + "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1552 + ", thread_n = " + str(th_config.thread_n) +
1553 + ", num_threads = " + str(th_config.num_threads) +
1554 + " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1555 + str(prob_n) + "]");
1556 +
1557 + int num_threads = th_config.num_threads;
1558 + thread_k = th_config.thread_k;
1559 + thread_n = th_config.thread_n;
1560 +
1561 + int thread_k_blocks = thread_k / 16;
1562 + int thread_n_blocks = thread_n / 16;
1563 +
1564 + int blocks = sms;
1565 +
1566 + TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
1567 + " is not divisible by thread_n = ", thread_n);
1568 + TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
1569 + " is not divisible by thread_k = ", thread_k);
1570 +
1571 + int group_blocks = 0;
1572 + if (has_act_order) {
1573 + if (is_k_full) {
1574 + TORCH_CHECK(group_size != -1);
1575 + group_blocks = group_size / 16;
1576 + TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1577 + " is not divisible by group_blocks = ", group_blocks);
1578 + } else {
1579 + TORCH_CHECK(group_size == 0);
1580 + group_blocks = 0;
1581 + }
1582 +
1583 + } else {
1584 + if (group_size == -1) {
1585 + group_blocks = -1;
1586 + } else {
1587 + group_blocks = group_size / 16;
1588 + TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1589 + " is not divisible by group_blocks = ", group_blocks);
1590 + }
1591 + }
1592 +
1593 + int max_shared_mem = 0;
1594 + cudaDeviceGetAttribute(&max_shared_mem,
1595 + cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1596 + TORCH_CHECK(max_shared_mem > 0);
1597 +
1598 + int tot_m = prob_m;
1599 +
1600 + const int* topk_ids_ptr = (const int*)topk_ids;
1601 + int* expert_offsets_ptr = (int*)expert_offsets;
1602 + compute_expert_offsets<<<1, num_experts, 0, stream>>>(
1603 + topk_ids_ptr, expert_offsets_ptr, tot_m * topk, moe_block_size);
1604 +
1605 + bool do_permute_a = has_act_order;
1606 +
1607 + // If we have a full K, then we can run the non-act-order version of Marlin
1608 + // (since the weight rows are reordered by increasing group ids, and by
1609 + // having a full K, we have full original groups)
1610 + if (is_k_full) {
1611 + has_act_order = false;
1612 + }
1613 +
1614 + for (int expert_idx = 0; expert_idx < num_experts; ++expert_idx) {
1615 + const int4* A_ptr = (const int4*)A;
1616 + int4* a_tmp_ptr = (int4*)a_tmp;
1617 + const int4* B_ptr = (const int4*)B + (prob_n * prob_k / 32) * expert_idx;
1618 + int4* C_ptr = (int4*)C;
1619 + const float* topk_weights_ptr = (const float*)topk_weights;
1620 + const int* sorted_ids_ptr = (const int*)sorted_ids;
1621 + const int4* s_ptr =
1622 + (const int4*)s +
1623 + (((group_size == -1 || group_size == 0) ? 1 : prob_k / group_size) *
1624 + prob_n / 8) *
1625 + expert_idx;
1626 + const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
1627 + const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
1628 + int* locks = (int*)workspace;
1629 +
1630 + if (do_permute_a) {
1631 + // Permute A columns
1632 + int topk_rows = replicate_input ? tot_m : tot_m * topk;
1633 + int block_rows = ceildiv(topk_rows, blocks);
1634 + permute_cols_kernel<<<blocks, num_threads, 0, stream>>>(
1635 + A_ptr, perm_ptr, a_tmp_ptr, topk_rows, prob_k, block_rows);
1636 + A_ptr = a_tmp_ptr;
1637 + }
1638 +
1639 + int max_m_blocks = ceildiv(tot_m, 16);
1640 + for (int m_block = 0; m_block < max_m_blocks; m_block += 16) {
1641 + // Define kernel configurations
1642 +
1643 + // make it max possible value
1644 + int thread_m_blocks = 4;
1645 +
1646 + if (false) {
1647 + }
1648 + CALL_IF_MOE(16, 4, 256)
1649 + CALL_IF_MOE(8, 8, 256)
1650 + CALL_IF_MOE(8, 4, 128)
1651 + CALL_IF_MOE(4, 8, 128)
1652 + else {
1653 + TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1654 + str(prob_n) + ", " + str(prob_k) + "]" +
1655 + ", has_act_order = " + str(has_act_order) +
1656 + ", num_groups = " + str(num_groups) +
1657 + ", group_size = " + str(group_size) +
1658 + ", thread_m_blocks = " + str(thread_m_blocks) +
1659 + ", thread_n_blocks = " + str(thread_n_blocks) +
1660 + ", thread_k_blocks = " + str(thread_k_blocks));
1661 + }
1662 + }
1663 + }
1664 +}
1665 +
1666 +} // namespace marlin_moe
1667 +
1668 +torch::Tensor marlin_gemm_moe(
1669 + const torch::Tensor& a, const torch::Tensor& b_q_weights,
1670 + const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
1671 + const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
1672 + const torch::Tensor& g_idx, const torch::Tensor& perm,
1673 + torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
1674 + bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
1675 + bool replicate_input, bool apply_weights) {
1676 + int max_par = 4;
1677 +
1678 + int dev = a.get_device();
1679 +
1680 + auto options_dtype =
1681 + torch::TensorOptions().dtype(a.dtype()).device(a.device());
1682 + auto options_int =
1683 + torch::TensorOptions().dtype(torch::kInt).device(a.device());
1684 + torch::Tensor c = torch::zeros({size_m, topk, size_n}, options_dtype);
1685 + torch::Tensor a_tmp =
1686 + replicate_input ? torch::zeros({size_m, size_k}, options_dtype)
1687 + : torch::zeros({size_m, topk, size_k}, options_dtype);
1688 + torch::Tensor expert_offsets = torch::empty({num_experts + 1}, options_int);
1689 +
1690 + // thread_k: `k` size of a thread_tile in `weights` (can usually be left as
1691 + // auto -1)
1692 + int thread_k = -1;
1693 + // thread_n: `n` size of a thread_tile in `weights` (can usually be left as
1694 + // auto -1)
1695 + int thread_n = -1;
1696 + // sms: number of SMs to use for the kernel (can usually be left as auto -1)
1697 + int sms = -1;
1698 +
1699 + // Detect groupsize and act_order
1700 + int num_groups = -1;
1701 + int group_size = -1;
1702 + bool has_act_order = g_idx.size(1) != 0;
1703 +
1704 + int b_rank = b_scales.sizes().size();
1705 + TORCH_CHECK(b_rank == 3, "b_scales rank = ", b_rank, " is not 3");
1706 + TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2),
1707 + " is not size_n = ", size_n);
1708 + num_groups = b_scales.size(1);
1709 +
1710 + if (has_act_order) {
1711 + if (is_k_full) {
1712 + TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
1713 + TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
1714 + ", is not divisible by num_groups = ", num_groups);
1715 + group_size = size_k / num_groups;
1716 + } else {
1717 + group_size = 0;
1718 + }
1719 +
1720 + } else {
1721 + if (num_groups > 1) {
1722 + TORCH_CHECK(
1723 + size_k % num_groups == 0, "size_k = ", size_k,
1724 + ", is not divisible by b_scales.size(0) = ", b_scales.size(0));
1725 + group_size = size_k / num_groups;
1726 + } else {
1727 + group_size = -1;
1728 + }
1729 + }
1730 +
1731 + marlin_moe::marlin_mm_moe_f16i4(
1732 + a.data_ptr(), b_q_weights.data_ptr(), c.data_ptr(), sorted_ids.data_ptr(),
1733 + topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
1734 + g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
1735 + expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
1736 + has_act_order, is_k_full, num_groups, group_size, num_experts, topk,
1737 + moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1738 + thread_n, sms, max_par, replicate_input, apply_weights);
1739 + return c;
1740 +}
\ No newline at end of file
csrc/moe/marlin_moe_ops.h new
+12
@@ -0,0 +1,12 @@
1 +#pragma once
2 +
3 +#include <torch/all.h>
4 +
5 +torch::Tensor marlin_gemm_moe(
6 + const torch::Tensor& a, const torch::Tensor& b_q_weights,
7 + const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
8 + const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
9 + const torch::Tensor& g_idx, const torch::Tensor& perm,
10 + torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
11 + bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
12 + bool replicate_input, bool apply_weights);
\ No newline at end of file
csrc/moe/torch_bindings.cpp
+9
@@ -1,5 +1,6 @@
1 #include "core/registration.h"
2 #include "moe_ops.h"
3 +#include "marlin_moe_ops.h"
4
5 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
6 // Apply topk softmax to the gating outputs.
@@ -7,6 +8,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
8 "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
9 "token_expert_indices, Tensor gating_output) -> ()");
10 m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
11 + m.def(
12 + "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
13 + "Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
14 + "g_idx, Tensor! perm, Tensor! workspace, int size_m, int size_n, int "
15 + "size_k, bool is_k_full, int num_experts, int topk, int moe_block_size, "
16 + "bool replicate_input, bool apply_weights) -> Tensor");
17 +
18 + m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
19 }
20
21 REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
tests/weight_loading/models.txt
+2
@@ -13,5 +13,7 @@ compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
13 compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
14 compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16 +compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
17 +compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
18 awq, casperhansen/mixtral-instruct-awq, main
19 awq_marlin, casperhansen/mixtral-instruct-awq, main
vllm/_custom_ops.py
+14
@@ -300,6 +300,20 @@ def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int,
300 return torch.ops._C.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits)
301
302
303 +def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
304 + size_k: int, size_n: int,
305 + num_bits: int) -> torch.Tensor:
306 + num_experts = b_q_weight.shape[0]
307 + assert size_k % 16 == 0
308 + output = torch.empty((num_experts, size_k // 16, size_n * 2),
309 + device=b_q_weight.device,
310 + dtype=b_q_weight.dtype)
311 + for e in range(num_experts):
312 + output[e] = torch.ops._C.gptq_marlin_repack(b_q_weight[e], perm[e],
313 + size_k, size_n, num_bits)
314 + return output
315 +
316 +
317 def gptq_marlin_gemm(a: torch.Tensor,
318 b_q_weight: torch.Tensor,
319 b_scales: torch.Tensor,
vllm/model_executor/layers/fused_moe/__init__.py
+6 -8
@@ -1,19 +1,17 @@
1 -from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
2 - FusedMoEMethodBase)
1 +from vllm.model_executor.layers.fused_moe.layer import (
2 + FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
3 from vllm.triton_utils import HAS_TRITON
4
5 -__all__ = [
6 - "FusedMoE",
7 - "FusedMoEMethodBase",
8 -]
5 +__all__ = ["FusedMoE", "FusedMoEMethodBase", "FusedMoeWeightScaleSupported"]
6
7 if HAS_TRITON:
8
9 from vllm.model_executor.layers.fused_moe.fused_moe import (
13 - fused_experts, fused_moe, fused_topk, get_config_file_name,
14 - grouped_topk)
10 + fused_experts, fused_marlin_moe, fused_moe, fused_topk,
11 + get_config_file_name, grouped_topk)
12
13 __all__ += [
14 + "fused_marlin_moe",
15 "fused_moe",
16 "fused_topk",
17 "fused_experts",
vllm/model_executor/layers/fused_moe/fused_moe.py
+116 -18
@@ -323,21 +323,16 @@ def get_moe_configs(E: int, N: int,
323 return None
324
325
326 -def get_default_config(
327 - M: int,
328 - E: int,
329 - N: int,
330 - K: int,
331 - topk: int,
332 - dtype: Optional[str],
333 -) -> Dict[str, int]:
326 +def get_default_config(M: int, E: int, N: int, K: int, topk: int,
327 + dtype: Optional[str],
328 + is_marlin: bool) -> Dict[str, int]:
329 config = {
330 'BLOCK_SIZE_M': 64,
331 'BLOCK_SIZE_N': 64,
332 'BLOCK_SIZE_K': 32,
333 'GROUP_SIZE_M': 8
334 }
340 - if M <= E:
335 + if M <= E or (is_marlin and M <= 32):
336 config = {
337 'BLOCK_SIZE_M': 16,
338 'BLOCK_SIZE_N': 32,
@@ -347,14 +342,14 @@ def get_default_config(
342 return config
343
344
350 -def try_get_optimal_moe_config(
351 - w1_shape: Tuple[int, ...],
352 - w2_shape: Tuple[int, ...],
353 - top_k: int,
354 - dtype: Optional[str],
355 - M: int,
356 - override_config: Optional[Dict[str, Any]] = None,
357 -):
345 +def try_get_optimal_moe_config(w1_shape: Tuple[int, ...],
346 + w2_shape: Tuple[int, ...],
347 + top_k: int,
348 + dtype: Optional[str],
349 + M: int,
350 + override_config: Optional[Dict[str,
351 + Any]] = None,
352 + is_marlin: bool = False):
353 if override_config:
354 config = override_config
355 else:
@@ -368,7 +363,8 @@ def try_get_optimal_moe_config(
363 config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
364 else:
365 # Else use the default config
371 - config = get_default_config(M, E, N, w1_shape[2], top_k, dtype)
366 + config = get_default_config(M, E, N, w1_shape[2], top_k, dtype,
367 + is_marlin)
368 return config
369
370
@@ -441,6 +437,108 @@ def grouped_topk(hidden_states: torch.Tensor,
437 return topk_weights, topk_ids
438
439
440 +def fused_marlin_moe(hidden_states: torch.Tensor,
441 + w1: torch.Tensor,
442 + w2: torch.Tensor,
443 + gating_output: torch.Tensor,
444 + g_idx1: torch.Tensor,
445 + g_idx2: torch.Tensor,
446 + rand_perm1: torch.Tensor,
447 + rand_perm2: torch.Tensor,
448 + topk: int,
449 + renormalize: bool,
450 + override_config: Optional[Dict[str, Any]] = None,
451 + use_fp8: bool = False,
452 + w1_scale: Optional[torch.Tensor] = None,
453 + w2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
454 + """
455 + This function computes a Mixture of Experts (MoE) layer using two sets of
456 + weights, w1 and w2, and top-k gating mechanism.
457 + Parameters:
458 + - hidden_states (torch.Tensor): The input tensor to the MoE layer.
459 + - w1 (torch.Tensor): The first set of expert weights.
460 + - w2 (torch.Tensor): The second set of expert weights.
461 + - gating_output (torch.Tensor): The output of the gating operation
462 + (before softmax).
463 + - topk (int): The number of top-k experts to select.
464 + - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
465 + - inplace (bool): If True, perform the operation in-place.
466 + Defaults to False.
467 + - override_config (Optional[Dict[str, Any]]): Optional override
468 + for the kernel configuration.
469 + - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
470 + products for w1 and w2. Defaults to False.
471 + - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
472 + w1.
473 + - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
474 + w2.
475 + Returns:
476 + - torch.Tensor: The output tensor after applying the MoE layer.
477 + """
478 + # Check constraints.
479 + assert hidden_states.shape[0] == gating_output.shape[0], (
480 + "Number of tokens mismatch")
481 + assert hidden_states.shape[
482 + 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
483 + assert hidden_states.shape[
484 + 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
485 + assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
486 + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
487 + assert w1.is_contiguous(), "Expert weights1 must be contiguous"
488 + assert w2.is_contiguous(), "Expert weights2 must be contiguous"
489 + assert hidden_states.dtype in [
490 + torch.float32, torch.float16, torch.bfloat16
491 + ]
492 +
493 + #TODO fp8 is not implemented yet
494 + assert not use_fp8
495 +
496 + M, K = hidden_states.shape
497 + E = w1.shape[0]
498 + N = w2.shape[1] * 16
499 +
500 + topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
501 + renormalize)
502 +
503 + get_config_func = functools.partial(try_get_optimal_moe_config,
504 + w1.shape,
505 + w2.shape,
506 + topk_ids.shape[1],
507 + "float8" if use_fp8 else None,
508 + override_config=override_config,
509 + is_marlin=True)
510 + config = get_config_func(M)
511 +
512 + block_size_m = config['BLOCK_SIZE_M']
513 +
514 + sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
515 +
516 + max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
517 + workspace = torch.zeros(max_workspace_size,
518 + dtype=torch.int,
519 + device="cuda",
520 + requires_grad=False)
521 +
522 + intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N),
523 + device=hidden_states.device,
524 + dtype=hidden_states.dtype)
525 +
526 + intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe(
527 + hidden_states, w1, sorted_token_ids, topk_weights, topk_ids, w1_scale,
528 + g_idx1, rand_perm1, workspace, M, 2 * N, K, True, E, topk,
529 + block_size_m, True, False)
530 +
531 + ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
532 +
533 + intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
534 + intermediate_cache2, w2, sorted_token_ids, topk_weights, topk_ids,
535 + w2_scale, g_idx2, rand_perm2, workspace, M, K, N, True, E, topk,
536 + block_size_m, False, True)
537 +
538 + return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
539 + dim=1)
540 +
541 +
542 def get_config_dtype_str(dtype: torch.dtype,
543 use_int8_w8a16: Optional[bool] = False,
544 use_fp8_w8a8: Optional[bool] = False):
vllm/model_executor/layers/fused_moe/layer.py
+170 -36
@@ -1,4 +1,5 @@
1 from abc import abstractmethod
2 +from enum import Enum
3 from typing import List, Optional, Tuple
4
5 import torch
@@ -15,6 +16,12 @@ from vllm.model_executor.utils import set_weight_attrs
16 logger = init_logger(__name__)
17
18
19 +class FusedMoeWeightScaleSupported(Enum):
20 + TENSOR = "tensor"
21 + CHANNEL = "channel"
22 + GROUP = "group"
23 +
24 +
25 class FusedMoEMethodBase(QuantizeMethodBase):
26
27 @abstractmethod
@@ -199,55 +206,182 @@ class FusedMoE(torch.nn.Module):
206 params_dtype=params_dtype,
207 weight_loader=self.weight_loader)
208
209 + def _load_per_tensor_weight_scale(self, shard_id: str,
210 + param: torch.nn.Parameter,
211 + loaded_weight: torch.Tensor,
212 + expert_id: int):
213 + param_data = param.data
214 + # for per tensor weight quantization
215 + if shard_id in ("w1", "w3"):
216 + # We have to keep the weight scales of w1 and w3 because
217 + # we need to re-quantize w1/w3 weights after weight loading.
218 + idx = 0 if shard_id == "w1" else 1
219 + param_data[expert_id][idx] = loaded_weight
220 + # If we are in the row parallel case (down_proj)
221 + elif shard_id == "w2":
222 + param_data[expert_id] = loaded_weight
223 +
224 + def _load_model_weight_or_group_weight_scale(self, shard_dim: int,
225 + expert_data: torch.Tensor,
226 + shard_id: str,
227 + loaded_weight: torch.tensor,
228 + tp_rank: int):
229 + # Load grouped weight scales for group quantization
230 + # or model weights
231 + if shard_id == "w2":
232 + self._load_w2(shard_id=shard_id,
233 + shard_dim=shard_dim,
234 + loaded_weight=loaded_weight,
235 + expert_data=expert_data,
236 + tp_rank=tp_rank)
237 + elif shard_id in ("w1", "w3"):
238 + self._load_w13(shard_id=shard_id,
239 + shard_dim=shard_dim,
240 + loaded_weight=loaded_weight,
241 + expert_data=expert_data,
242 + tp_rank=tp_rank)
243 +
244 + def _load_per_channel_weight_scale(self, expert_data: torch.Tensor,
245 + shard_dim: int, shard_id: str,
246 + loaded_weight: torch.tensor,
247 + tp_rank: int):
248 + # for per channel weight quantization
249 + if shard_id == "w2":
250 + expert_data.copy_(loaded_weight)
251 + elif shard_id in ("w1", "w3"):
252 + self._load_w13(shard_id=shard_id,
253 + shard_dim=shard_dim,
254 + loaded_weight=loaded_weight,
255 + expert_data=expert_data,
256 + tp_rank=tp_rank)
257 +
258 + def _load_w13(self, expert_data: torch.Tensor, shard_dim: int,
259 + shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
260 +
261 + # Index the loaded weight for tp sharding.
262 + # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
263 + shard_size = expert_data.shape[shard_dim] // 2
264 + loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
265 + shard_size)
266 + # Narrow parameter and load.
267 + # w1, gate_proj: Load into first logical weight of w13.
268 + if shard_id == "w1":
269 + expert_data = expert_data.narrow(shard_dim, 0, shard_size)
270 + # w3, up_proj: Load into second logical weight of w13.
271 + else:
272 + assert shard_id == "w3"
273 + expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
274 + expert_data.copy_(loaded_weight)
275 +
276 + def _load_w2(self, expert_data: torch.Tensor, shard_dim: int,
277 + shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
278 +
279 + # Index the loaded weight for tp sharding.
280 + # down_proj: "RowParallel" so tp sharding on input_dim
281 + # Narrow parameter and load.
282 + shard_size = expert_data.shape[shard_dim]
283 + loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
284 + shard_size)
285 + # w2, down_proj: Load into only logical weight of w2.
286 + expert_data.copy_(loaded_weight)
287 +
288 + def _load_single_value(self, param: torch.nn.Parameter,
289 + loaded_weight: torch.Tensor, expert_id: int):
290 + param_data = param.data
291 +
292 + # Input scales can be loaded directly and should be equal.
293 + param_data[expert_id] = loaded_weight
294 +
295 def weight_loader(self, param: torch.nn.Parameter,
296 loaded_weight: torch.Tensor, weight_name: str,
297 shard_id: str, expert_id: int) -> None:
298 +
299 if shard_id not in ("w1", "w2", "w3"):
300 raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
301 f"got {shard_id}.")
302
209 - # Special case for fp8 scales.
210 - if getattr(param, "is_fp8_scale", False):
211 - self._load_fp8_scale(param.data, loaded_weight, weight_name,
212 - shard_id, expert_id)
213 - return
303 + WEIGHT_SCALE_SUPPORTED = [
304 + e.value for e in FusedMoeWeightScaleSupported
305 + ]
306 + # Fetch the dim to shard the parameter/loaded weight
307 + # based on the shard id. This will be whatever
308 + # dimension intermediate_size is used.
309 + SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
310
311 expert_data = param.data[expert_id]
312 tp_rank = get_tensor_model_parallel_rank()
313
218 - # If transposed, weight is saved as [input_dim, output_dim]
219 - # Otherwise, weight is saved as [output_dim, input_dim]
220 - # Default is not transposed/input dim is dim 1
221 - input_dim = getattr(param, "input_dim", 1)
222 - output_dim = getattr(param, "output_dim", 0)
314 + # is_transposed: whether or not the parameter is transposed on disk
315 + # If transposed, the loaded weight will be transposed and the dim
316 + # to shard the loaded weight will be flipped.
317 + is_transposed = getattr(param, "is_transposed", False)
318 + shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
319 + if is_transposed:
320 + loaded_weight = loaded_weight.t().contiguous()
321 + shard_dim = ~shard_dim
322 +
323 + # Case weight_scales
324 + if "weight_scale" in weight_name:
325 + # load the weight scaling based on the quantization scheme
326 + # supported weight scales can be found in
327 + # FusedMoeWeightScaleSupported
328 + # TODO @dsikka: once hardened, refactor to use vLLM Parameters
329 + # specific to each case
330 + quant_method = getattr(param, "quant_method", None)
331 + if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value:
332 + self._load_per_channel_weight_scale(
333 + shard_id=shard_id,
334 + shard_dim=shard_dim,
335 + loaded_weight=loaded_weight,
336 + expert_data=expert_data,
337 + tp_rank=tp_rank)
338 + elif quant_method == FusedMoeWeightScaleSupported.GROUP.value:
339 + self._load_model_weight_or_group_weight_scale(
340 + shard_id=shard_id,
341 + shard_dim=shard_dim,
342 + loaded_weight=loaded_weight,
343 + expert_data=expert_data,
344 + tp_rank=tp_rank)
345 + elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
346 + self._load_per_tensor_weight_scale(shard_id=shard_id,
347 + param=param,
348 + loaded_weight=loaded_weight,
349 + expert_id=expert_id)
350 + else:
351 + raise ValueError(
352 + f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}")
353 + return
354
224 - # Index the loaded weight for tp sharding.
225 - # down_proj: "RowParallel" so tp sharding on input_dim
226 - if shard_id == "w2":
227 - shard_dim = input_dim
228 - shard_size = expert_data.shape[shard_dim]
229 - # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
230 - elif shard_id in ("w1", "w3"):
231 - shard_dim = output_dim
232 - shard_size = expert_data.shape[output_dim] // 2
233 - offset = shard_size * tp_rank
234 - loaded_weight = loaded_weight.narrow(shard_dim, offset, shard_size)
355 + if "weight_shape" in weight_name:
356 + self._load_single_value(param=param,
357 + loaded_weight=loaded_weight,
358 + expert_id=expert_id)
359 + return
360
236 - # Narrow parameter and load.
237 - # w1, gate_proj: Load into first logical weight of w13.
238 - if shard_id == "w1":
239 - expert_data = expert_data.narrow(shard_dim, 0, shard_size)
240 - expert_data.copy_(loaded_weight)
241 - # w3, up_proj: Load into second logical weight of w13.
242 - elif shard_id == "w3":
243 - expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
244 - expert_data.copy_(loaded_weight)
245 - # w2, down_proj: Load into only logical weight of w2.
246 - elif shard_id == "w2":
247 - expert_data.copy_(loaded_weight)
248 - else:
249 - raise ValueError(
250 - f"Expected shard_id w1,w2 or w3 but got {shard_id}")
361 + # Case input scale
362 + if "input_scale" in weight_name:
363 + # Note: input_scale loading is only supported for fp8
364 + if param.data[expert_id] != 1 and (param.data[expert_id] -
365 + loaded_weight).abs() > 1e-5:
366 + raise ValueError(
367 + "input_scales of w1 and w3 of a layer "
368 + f"must be equal. But got {param.data[expert_id]} "
369 + f"vs. {loaded_weight}")
370 +
371 + self._load_single_value(param=param,
372 + loaded_weight=loaded_weight,
373 + expert_id=expert_id)
374 + return
375 +
376 + # Case model weights
377 + if "weight" in weight_name:
378 + self._load_model_weight_or_group_weight_scale(
379 + shard_id=shard_id,
380 + shard_dim=shard_dim,
381 + loaded_weight=loaded_weight,
382 + expert_data=expert_data,
383 + tp_rank=tp_rank)
384 + return
385
386 @staticmethod
387 def select_experts(hidden_states: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+5
@@ -3,9 +3,12 @@ from typing import Any, Dict, List, Optional
3 import torch
4 from pydantic import BaseModel
5
6 +from vllm.model_executor.layers.fused_moe import FusedMoE
7 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
9 QuantizationConfig, QuantizeMethodBase)
10 +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
11 + CompressedTensorsMoEMethod)
12 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
13 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
14 CompressedTensorsScheme, CompressedTensorsUnquantized,
@@ -64,6 +67,8 @@ class CompressedTensorsConfig(QuantizationConfig):
67 return CompressedTensorsLinearMethod(self)
68 if isinstance(layer, Attention):
69 return CompressedTensorsKVCacheMethod(self)
70 + if isinstance(layer, FusedMoE):
71 + return CompressedTensorsMoEMethod(self)
72 return None
73
74 @classmethod
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py new
+283
@@ -0,0 +1,283 @@
1 +import enum
2 +from enum import Enum
3 +from typing import List, Optional
4 +
5 +import torch
6 +
7 +from vllm import _custom_ops as ops
8 +from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
9 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 + WNA16_SUPPORTED_BITS)
11 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
12 + CompressionFormat)
13 +from vllm.model_executor.utils import set_weight_attrs
14 +
15 +
16 +class GPTQMarlinState(Enum):
17 + REPACK = enum.auto()
18 + READY = enum.auto()
19 +
20 +
21 +__all__ = ["CompressedTensorsMoEMethod"]
22 +
23 +
24 +class CompressedTensorsMoEMethod(FusedMoEMethodBase):
25 +
26 + def __init__(
27 + self,
28 + quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501
29 + ):
30 + self.quant_config = quant_config
31 + # TODO: @dsikka: refactor this to use schemes as other kernels
32 + # are supported + check if the layer is being ignored.
33 + config = self.quant_config.target_scheme_map["Linear"].get("weights")
34 + self.num_bits = config.num_bits
35 + self.packed_factor = 32 // config.num_bits
36 + self.strategy = config.strategy.value
37 + self.group_size = config.group_size
38 + assert config.symmetric, (
39 + "Only symmetric quantization is supported for MoE")
40 +
41 + if not (self.quant_config.quant_format
42 + == CompressionFormat.pack_quantized.value
43 + and self.num_bits in WNA16_SUPPORTED_BITS):
44 + raise ValueError("For Fused MoE layers, only ",
45 + f"{CompressionFormat.pack_quantized.value} ",
46 + "is supported for the following bits: ",
47 + f"{WNA16_SUPPORTED_BITS}")
48 +
49 + def create_weights(self, layer: torch.nn.Module, num_experts: int,
50 + hidden_size: int, intermediate_size: int,
51 + params_dtype: torch.dtype, **extra_weight_attrs):
52 +
53 + # Will transpose the loaded weight along the
54 + # intermediate and hidden dim sizes. Will
55 + # shard for TP along the transposed dims
56 + extra_weight_attrs.update({
57 + "is_transposed": True,
58 + "quant_method": self.strategy
59 + })
60 + w13_weight = torch.nn.Parameter(torch.empty(num_experts,
61 + hidden_size //
62 + self.packed_factor,
63 + 2 * intermediate_size,
64 + dtype=torch.int32),
65 + requires_grad=False)
66 + layer.register_parameter("w13_weight_packed", w13_weight)
67 + set_weight_attrs(w13_weight, extra_weight_attrs)
68 +
69 + w2_weight = torch.nn.Parameter(torch.empty(num_experts,
70 + intermediate_size //
71 + self.packed_factor,
72 + hidden_size,
73 + dtype=torch.int32),
74 + requires_grad=False)
75 + layer.register_parameter("w2_weight_packed", w2_weight)
76 + set_weight_attrs(w2_weight, extra_weight_attrs)
77 +
78 + if self.strategy == "channel":
79 + num_groups_w2 = num_groups_w13 = 1
80 + self.group_size = -1
81 + else:
82 + num_groups_w2 = intermediate_size // self.group_size
83 + num_groups_w13 = hidden_size // self.group_size
84 +
85 + w13_scale = torch.nn.Parameter(torch.ones(num_experts,
86 + num_groups_w13,
87 + 2 * intermediate_size,
88 + dtype=params_dtype),
89 + requires_grad=False)
90 + layer.register_parameter("w13_weight_scale", w13_scale)
91 + set_weight_attrs(w13_scale, extra_weight_attrs)
92 +
93 + w2_scale = torch.nn.Parameter(torch.ones(num_experts,
94 + num_groups_w2,
95 + hidden_size,
96 + dtype=params_dtype),
97 + requires_grad=False)
98 + layer.register_parameter("w2_weight_scale", w2_scale)
99 + set_weight_attrs(w2_scale, extra_weight_attrs)
100 +
101 + w2_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
102 + requires_grad=False)
103 + layer.register_parameter("w2_weight_shape", w2_weight_shape)
104 + set_weight_attrs(w2_weight_shape, extra_weight_attrs)
105 + w13_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
106 + requires_grad=False)
107 +
108 + layer.register_parameter("w13_weight_shape", w13_weight_shape)
109 + set_weight_attrs(w13_weight_shape, extra_weight_attrs)
110 +
111 + w13_g_idx = torch.nn.Parameter(
112 + torch.empty(
113 + num_experts,
114 + hidden_size,
115 + dtype=torch.int32,
116 + ),
117 + requires_grad=False,
118 + )
119 + layer.register_parameter("w13_g_idx", w13_g_idx)
120 + set_weight_attrs(w13_g_idx, extra_weight_attrs)
121 +
122 + w2_g_idx = torch.nn.Parameter(
123 + torch.empty(
124 + num_experts,
125 + intermediate_size,
126 + dtype=torch.int32,
127 + ),
128 + requires_grad=False,
129 + )
130 + layer.register_parameter("w2_g_idx", w2_g_idx)
131 + set_weight_attrs(w2_g_idx, extra_weight_attrs)
132 +
133 + w13_g_idx_sort_indices = torch.nn.Parameter(
134 + torch.empty(
135 + num_experts,
136 + hidden_size,
137 + dtype=torch.int32,
138 + ),
139 + requires_grad=False,
140 + )
141 + layer.register_parameter("w13_g_idx_sort_indices",
142 + w13_g_idx_sort_indices)
143 + set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
144 +
145 + w2_g_idx_sort_indices = torch.nn.Parameter(
146 + torch.empty(
147 + num_experts,
148 + intermediate_size,
149 + dtype=torch.int32,
150 + ),
151 + requires_grad=False,
152 + )
153 + layer.register_parameter("w2_g_idx_sort_indices",
154 + w2_g_idx_sort_indices)
155 + set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
156 +
157 + layer.a13_scale = None
158 + layer.a2_scale = None
159 + layer.marlin_state = GPTQMarlinState.REPACK
160 +
161 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
162 +
163 + def replace_tensor(name, new_t):
164 + # It is important to use resize_() here since it ensures
165 + # the same buffer is reused
166 + getattr(layer, name).resize_(new_t.shape)
167 + getattr(layer, name).copy_(new_t)
168 + del new_t
169 +
170 + def get_scale_perms(num_bits: int):
171 + scale_perm: List[int] = []
172 + for i in range(8):
173 + scale_perm.extend([i + 8 * j for j in range(8)])
174 + scale_perm_single: List[int] = []
175 + for i in range(4):
176 + scale_perm_single.extend(
177 + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
178 + return scale_perm, scale_perm_single
179 +
180 + def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
181 + group_size: int, num_bits: int):
182 + scale_perm, scale_perm_single = get_scale_perms(num_bits)
183 + if group_size < size_k and group_size != -1:
184 + s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
185 + else:
186 + s = s.reshape((-1, len(scale_perm_single)))[:,
187 + scale_perm_single]
188 + s = s.reshape((-1, size_n)).contiguous()
189 + return s
190 +
191 + def marlin_moe_permute_scales(s: torch.Tensor, size_k: int,
192 + size_n: int, group_size: int,
193 + num_bits: int):
194 + num_experts = s.shape[0]
195 + output = torch.empty((num_experts, s.shape[1], s.shape[2]),
196 + device=s.device,
197 + dtype=s.dtype)
198 + for e in range(num_experts):
199 + output[e] = marlin_permute_scales(s[e], size_k, size_n,
200 + group_size, num_bits)
201 + return output
202 +
203 + size_k2 = layer.w2_weight_packed.shape[2]
204 + size_k13 = layer.w13_weight_packed.shape[2]
205 +
206 + num_experts = layer.w13_g_idx.shape[0]
207 + device = layer.w13_g_idx.device
208 + layer.w13_g_idx = torch.nn.Parameter(
209 + torch.empty((num_experts, 0), dtype=torch.int32, device=device),
210 + requires_grad=False,
211 + )
212 + layer.w2_g_idx = torch.nn.Parameter(
213 + torch.empty((num_experts, 0), dtype=torch.int32, device=device),
214 + requires_grad=False,
215 + )
216 + layer.w13_g_idx_sort_indices = torch.nn.Parameter(
217 + torch.empty((num_experts, 0), dtype=torch.int32, device=device),
218 + requires_grad=False,
219 + )
220 + layer.w2_g_idx_sort_indices = torch.nn.Parameter(
221 + torch.empty((num_experts, 0), dtype=torch.int32, device=device),
222 + requires_grad=False,
223 + )
224 +
225 + marlin_w13_qweight = ops.gptq_marlin_moe_repack(
226 + layer.w13_weight_packed,
227 + layer.w13_g_idx_sort_indices,
228 + layer.w13_weight_packed.shape[1] * self.packed_factor,
229 + layer.w13_weight_packed.shape[2],
230 + self.num_bits,
231 + )
232 + replace_tensor("w13_weight_packed", marlin_w13_qweight)
233 + marlin_w2_qweight = ops.gptq_marlin_moe_repack(
234 + layer.w2_weight_packed,
235 + layer.w2_g_idx_sort_indices,
236 + layer.w2_weight_packed.shape[1] * self.packed_factor,
237 + layer.w2_weight_packed.shape[2],
238 + self.num_bits,
239 + )
240 + replace_tensor("w2_weight_packed", marlin_w2_qweight)
241 + # Repack scales
242 + marlin_w13_scales = marlin_moe_permute_scales(
243 + layer.w13_weight_scale,
244 + size_k13,
245 + layer.w13_weight_scale.shape[2],
246 + self.group_size,
247 + self.num_bits,
248 + )
249 + replace_tensor("w13_weight_scale", marlin_w13_scales)
250 + marlin_w2_scales = marlin_moe_permute_scales(
251 + layer.w2_weight_scale,
252 + layer.w2_weight_scale.shape[1] * self.packed_factor,
253 + size_k2,
254 + self.group_size,
255 + self.num_bits,
256 + )
257 + replace_tensor("w2_weight_scale", marlin_w2_scales)
258 +
259 + def apply(self,
260 + layer: torch.nn.Module,
261 + x: torch.Tensor,
262 + router_logits: torch.Tensor,
263 + top_k: int,
264 + renormalize: bool = True,
265 + use_grouped_topk: bool = False,
266 + num_expert_group: Optional[int] = None,
267 + topk_group: Optional[int] = None) -> torch.Tensor:
268 +
269 + from vllm.model_executor.layers.fused_moe.fused_moe import (
270 + fused_marlin_moe)
271 +
272 + return fused_marlin_moe(x,
273 + layer.w13_weight_packed,
274 + layer.w2_weight_packed,
275 + router_logits,
276 + layer.w13_g_idx,
277 + layer.w2_g_idx,
278 + layer.w13_g_idx_sort_indices,
279 + layer.w2_g_idx_sort_indices,
280 + top_k,
281 + renormalize=renormalize,
282 + w1_scale=layer.w13_weight_scale,
283 + w2_scale=layer.w2_weight_scale)
vllm/model_executor/layers/quantization/fp8.py
+11 -18
@@ -7,7 +7,8 @@ from torch.nn.parameter import Parameter
7 import vllm.envs as envs
8 from vllm import _custom_ops as ops
9 from vllm.logger import init_logger
10 -from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
10 +from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11 + FusedMoeWeightScaleSupported)
12 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
13 UnquantizedLinearMethod)
14 from vllm.model_executor.layers.quantization.base_config import (
@@ -318,19 +319,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
319 dtype=torch.float32),
320 requires_grad=False)
321 layer.register_parameter("w2_weight_scale", w2_weight_scale)
321 -
322 + # Add the quantization method used (per tensor/grouped/channel)
323 + # to ensure the weight scales are loaded in properly
324 + extra_weight_attrs.update(
325 + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
326 # If loading fp8 checkpoint, pass the weight loaders.
327 # If loading an fp16 checkpoint, do not (we will quantize in
328 # process_weights_after_loading()
329 if self.quant_config.is_checkpoint_fp8_serialized:
326 - set_weight_attrs(w13_weight_scale, {
327 - "is_fp8_scale": True,
328 - **extra_weight_attrs
329 - })
330 - set_weight_attrs(w2_weight_scale, {
331 - "is_fp8_scale": True,
332 - **extra_weight_attrs
333 - })
330 + set_weight_attrs(w13_weight_scale, extra_weight_attrs)
331 + set_weight_attrs(w2_weight_scale, extra_weight_attrs)
332
333 # INPUT_SCALES
334 if self.quant_config.activation_scheme == "static":
@@ -343,19 +341,14 @@ class Fp8MoEMethod(FusedMoEMethodBase):
341 num_experts, dtype=torch.float32),
342 requires_grad=False)
343 layer.register_parameter("w13_input_scale", w13_input_scale)
346 - set_weight_attrs(w13_input_scale, {
347 - "is_fp8_scale": True,
348 - **extra_weight_attrs
349 - })
344 + set_weight_attrs(w13_input_scale, extra_weight_attrs)
345
346 w2_input_scale = torch.nn.Parameter(torch.ones(
347 num_experts, dtype=torch.float32),
348 requires_grad=False)
349 layer.register_parameter("w2_input_scale", w2_input_scale)
355 - set_weight_attrs(w2_input_scale, {
356 - "is_fp8_scale": True,
357 - **extra_weight_attrs
358 - })
350 + set_weight_attrs(w2_input_scale, extra_weight_attrs)
351 +
352 else:
353 layer.w13_input_scale = None
354 layer.w2_input_scale = None
vllm/model_executor/model_loader/utils.py
+2 -2
@@ -23,11 +23,11 @@ def get_model_architecture(
23 architectures = getattr(model_config.hf_config, "architectures", [])
24 # Special handling for quantized Mixtral.
25 # FIXME(woosuk): This is a temporary hack.
26 + mixtral_supported = ["fp8", "compressed-tensors"]
27 if (model_config.quantization is not None
27 - and model_config.quantization != "fp8"
28 + and model_config.quantization not in mixtral_supported
29 and "MixtralForCausalLM" in architectures):
30 architectures = ["QuantMixtralForCausalLM"]
30 -
31 return ModelRegistry.resolve_model_cls(architectures)
32
33
vllm/model_executor/models/jamba.py
+1 -1
@@ -920,7 +920,7 @@ class JambaForCausalLM(nn.Module, HasInnerState):
920 weight_loader = param.weight_loader
921 weight_loader(param,
922 loaded_weight,
923 - weight_name,
923 + name,
924 shard_id=shard_id,
925 expert_id=expert_id)
926 break
vllm/model_executor/models/mixtral.py
+1
@@ -73,6 +73,7 @@ class MixtralMoE(nn.Module):
73 self.hidden_size = hidden_size
74
75 # Gate always runs at half / full precision for now.
76 +
77 self.gate = ReplicatedLinear(hidden_size,
78 num_experts,
79 bias=False,