@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aae74ef95c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

Revert "[Kernel] Expand MoE weight loading + Add Fused Marlin MoE Kernel (#7527)" (#7764)

Michael Goin committed Aug 21, 2024 at 23:42 UTC aae74ef95c370df92584e08939a15707ea8a5d3f
15 files changed +84 -2374
CMakeLists.txt
+1 -2
@@ -286,8 +286,7 @@ define_gpu_extension_target(
286
287 set(VLLM_MOE_EXT_SRC
288 "csrc/moe/torch_bindings.cpp"
289 - "csrc/moe/topk_softmax_kernels.cu"
290 - "csrc/moe/marlin_moe_ops.cu")
289 + "csrc/moe/topk_softmax_kernels.cu")
290
291 define_gpu_extension_target(
292 _moe_C
csrc/moe/marlin_moe_ops.cu deleted
-1740
@@ -1,1740 +0,0 @@
1 -/*
2 - * Modified by Neural Magic
3 - * Copyright (C) Marlin.2024 Elias Frantar
4 - *
5 - * Licensed under the Apache License, Version 2.0 (the "License");
6 - * you may not use this file except in compliance with the License.
7 - * You may obtain a copy of the License at
8 - *
9 - * http://www.apache.org/licenses/LICENSE-2.0
10 - *
11 - * Unless required by applicable law or agreed to in writing, software
12 - * distributed under the License is distributed on an "AS IS" BASIS,
13 - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 - * See the License for the specific language governing permissions and
15 - * limitations under the License.
16 - */
17 -
18 -#include <torch/all.h>
19 -
20 -#include <ATen/cuda/CUDAContext.h>
21 -#include <c10/cuda/CUDAGuard.h>
22 -#include <cuda.h>
23 -#include <cuda_fp16.h>
24 -#include <cuda_runtime.h>
25 -
26 -#include <iostream>
27 -
28 -template <typename T>
29 -inline std::string str(T x) {
30 - return std::to_string(x);
31 -}
32 -
33 -namespace marlin_moe {
34 -
35 -constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
36 -
37 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
38 -
39 -// Instances of `Vec` are used to organize groups of >>registers<<, as needed
40 -// for instance as inputs to tensor core operations. Consequently, all
41 -// corresponding index accesses must be compile-time constants, which is why we
42 -// extensively use `#pragma unroll` throughout the kernel code to guarantee
43 -// this.
44 -template <typename T, int n>
45 -struct Vec {
46 - T elems[n];
47 - __device__ T& operator[](int i) { return elems[i]; }
48 -};
49 -
50 -using I4 = Vec<int, 4>;
51 -
52 -// Matrix fragments for tensor core instructions; their precise layout is
53 -// documented here:
54 -// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
55 -using FragA = Vec<half2, 4>;
56 -using FragB = Vec<half2, 2>;
57 -using FragC = Vec<float, 4>;
58 -using FragS = Vec<half2, 1>; // quantization scales
59 -
60 -// Predicated asynchronous global->shared copy; used for inputs A where we apply
61 -// predication to handle batchsizes that are not multiples of 16.
62 -__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
63 - bool pred = true) {
64 - const int BYTES = 16;
65 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
66 - asm volatile(
67 - "{\n"
68 - " .reg .pred p;\n"
69 - " setp.ne.b32 p, %0, 0;\n"
70 - " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
71 - "}\n" ::"r"((int)pred),
72 - "r"(smem), "l"(glob_ptr), "n"(BYTES));
73 -}
74 -
75 -// Asynchronous global->shared copy
76 -__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
77 - const int BYTES = 16;
78 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
79 - asm volatile(
80 - "{\n"
81 - " cp.async.cg.shared.global [%0], [%1], %2;\n"
82 - "}\n" ::"r"(smem),
83 - "l"(glob_ptr), "n"(BYTES));
84 -}
85 -
86 -// Async copy fence.
87 -__device__ inline void cp_async_fence() {
88 - asm volatile("cp.async.commit_group;\n" ::);
89 -}
90 -
91 -// Wait until at most `n` async copy stages are still pending.
92 -template <int n>
93 -__device__ inline void cp_async_wait() {
94 - asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
95 -}
96 -
97 -// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
98 -// output/accumulation.
99 -__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
100 - FragC& frag_c) {
101 - const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
102 - const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
103 - float* c = reinterpret_cast<float*>(&frag_c);
104 - asm volatile(
105 - "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
106 - "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
107 - : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
108 - : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
109 - "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
110 -}
111 -
112 -// Instruction for loading a full 16x16 matrix fragment of operand A from shared
113 -// memory, directly in tensor core layout.
114 -__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
115 - uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
116 - uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
117 - asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
118 - : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
119 - : "r"(smem));
120 -}
121 -
122 -// Lookup-table based 3-input logical operation; explicitly used for
123 -// dequantization as the compiler does not seem to automatically recognize it in
124 -// all cases.
125 -template <int lut>
126 -__device__ inline int lop3(int a, int b, int c) {
127 - int res;
128 - asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
129 - : "=r"(res)
130 - : "r"(a), "r"(b), "r"(c), "n"(lut));
131 - return res;
132 -}
133 -
134 -// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
135 -// values. We mostly follow the strategy in the link below, with some small
136 -// changes:
137 -// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
138 -__device__ inline FragB dequant(int q) {
139 - const int LO = 0x000f000f;
140 - const int HI = 0x00f000f0;
141 - const int EX = 0x64006400;
142 - // Guarantee that the `(a & b) | c` operations are LOP3s.
143 - int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
144 - int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
145 - // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
146 - // directly into `SUB` and `ADD`.
147 - const int SUB = 0x64086408;
148 - const int MUL = 0x2c002c00;
149 - const int ADD = 0xd480d480;
150 - FragB frag_b;
151 - frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
152 - *reinterpret_cast<const half2*>(&SUB));
153 - frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
154 - *reinterpret_cast<const half2*>(&MUL),
155 - *reinterpret_cast<const half2*>(&ADD));
156 - return frag_b;
157 -}
158 -
159 -// Multiply dequantized values by the corresponding quantization scale; used
160 -// only for grouped quantization.
161 -__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
162 - half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
163 - frag_b[0] = __hmul2(frag_b[0], s);
164 - frag_b[1] = __hmul2(frag_b[1], s);
165 -}
166 -
167 -// Given 2 floats multiply by 2 scales (halves)
168 -__device__ inline void scale_float(float* c, FragS& s) {
169 - __half* s_ptr = reinterpret_cast<__half*>(&s);
170 - c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
171 - c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
172 -}
173 -
174 -// Same as above, but for act_order (each K is multiplied individually)
175 -__device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
176 - FragS& frag_s_3, FragS& frag_s_4, int i) {
177 - __half2 s_val_1_2;
178 - s_val_1_2.x = reinterpret_cast<__half*>(&frag_s_1)[i];
179 - s_val_1_2.y = reinterpret_cast<__half*>(&frag_s_2)[i];
180 -
181 - __half2 s_val_3_4;
182 - s_val_3_4.x = reinterpret_cast<__half*>(&frag_s_3)[i];
183 - s_val_3_4.y = reinterpret_cast<__half*>(&frag_s_4)[i];
184 -
185 - frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
186 - frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
187 -}
188 -
189 -// Wait until barrier reaches `count`, then lock for current threadblock.
190 -__device__ inline void barrier_acquire(int* lock, int count) {
191 - if (threadIdx.x == 0) {
192 - int state = -1;
193 - do
194 - // Guarantee that subsequent writes by this threadblock will be visible
195 - // globally.
196 - asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
197 - : "=r"(state)
198 - : "l"(lock));
199 - while (state != count);
200 - }
201 - __syncthreads();
202 -}
203 -
204 -// Release barrier and increment visitation count.
205 -__device__ inline void barrier_release(int* lock, bool reset = false) {
206 - __syncthreads();
207 - if (threadIdx.x == 0) {
208 - if (reset) {
209 - lock[0] = 0;
210 - return;
211 - }
212 - int val = 1;
213 - // Make sure that all writes since acquiring this barrier are visible
214 - // globally, while releasing the barrier.
215 - asm volatile("fence.acq_rel.gpu;\n");
216 - asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
217 - :
218 - : "l"(lock), "r"(val));
219 - }
220 -}
221 -
222 -// For a given "a" of size [M,K] performs a permutation of the K columns based
223 -// on the given "perm" indices.
224 -__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
225 - int const* __restrict__ perm_int_ptr,
226 - int4* __restrict__ out_int4_ptr, int size_m,
227 - int size_k, int block_rows) {
228 - int start_row = block_rows * blockIdx.x;
229 - int finish_row = start_row + block_rows;
230 - if (finish_row > size_m) {
231 - finish_row = size_m;
232 - }
233 - int cur_block_rows = finish_row - start_row;
234 -
235 - int row_stride = size_k * sizeof(half) / 16;
236 -
237 - auto permute_row = [&](int row) {
238 - int iters = size_k / blockDim.x;
239 - int rest = size_k % blockDim.x;
240 -
241 - int offset = row * row_stride;
242 -
243 - half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
244 - half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
245 -
246 - int base_k = 0;
247 -
248 - for (int i = 0; i < iters; i++) {
249 - int cur_k = base_k + threadIdx.x;
250 - int src_pos = perm_int_ptr[cur_k];
251 -
252 - out_half[cur_k] = a_row_half[src_pos];
253 -
254 - base_k += blockDim.x;
255 - }
256 -
257 - if (rest) {
258 - if (threadIdx.x < rest) {
259 - int cur_k = base_k + threadIdx.x;
260 - int src_pos = perm_int_ptr[cur_k];
261 -
262 - out_half[cur_k] = a_row_half[src_pos];
263 - }
264 - }
265 - };
266 -
267 - for (int i = 0; i < cur_block_rows; i++) {
268 - int cur_row = start_row + i;
269 - if (cur_row < size_m) {
270 - permute_row(cur_row);
271 - }
272 - }
273 -}
274 -
275 -__global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
276 - int* __restrict__ expert_offsets,
277 - int topk_length, int block_size) {
278 - int expert_id = threadIdx.x;
279 - int num_experts = blockDim.x;
280 -
281 - int occurrences = 0;
282 - for (int i = 0; i < topk_length; ++i) {
283 - occurrences += (topk_ids[i] == expert_id);
284 - }
285 - expert_offsets[expert_id + 1] = occurrences;
286 - __syncthreads();
287 -
288 - if (threadIdx.x == 0) {
289 - int tot_offset = 0;
290 - expert_offsets[0] = 0;
291 - for (int i = 0; i < num_experts; ++i) {
292 - tot_offset += ceildiv(expert_offsets[i + 1], block_size) * block_size;
293 - expert_offsets[i + 1] = tot_offset;
294 - }
295 - }
296 - __syncthreads();
297 -}
298 -
299 -template <const int threads, // number of threads in a threadblock
300 - const int thread_m_blocks, // number of 16x16 blocks in the m
301 - // dimension (batchsize) of the
302 - // threadblock
303 - const int thread_n_blocks, // same for n dimension (output)
304 - const int thread_k_blocks, // same for k dimension (reduction)
305 - const int stages, // number of stages for the async global->shared
306 - // fetch pipeline
307 - const bool has_act_order, // whether act_order is enabled
308 - const int group_blocks = -1 // number of consecutive 16x16 blocks
309 - // with a separate quantization scale
310 - >
311 -__device__ inline void MarlinMoESingle(
312 - const int4* __restrict__ A, // fp16 input matrix of shape mxk
313 - const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
314 - int4* __restrict__ C, // fp16 output buffer of shape mxn
315 - const int* __restrict__ sorted_ids, // int32 sorted ids of experts
316 - const float* __restrict__ topk_weights, // float topk weights
317 - const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
318 - // (k/groupsize)xn
319 - const int* __restrict__ g_idx, // int32 group indices of shape k
320 - const int* __restrict__ expert_offsets,
321 - int num_groups, // number of scale groups per output channel
322 - int expert_idx, // idx of current expert
323 - int num_experts, // number of experts
324 - int topk, // topk parameter of moe
325 - int prob_m, // batch dimension m
326 - int prob_n, // output dimension n
327 - int prob_k, // reduction dimension k
328 - int tot_m, // total number of rows in A and C
329 - int* locks, // extra global storage for barrier synchronization
330 - bool replicate_input, // do we use the same input for each expert?
331 - bool apply_weights, // apply weights to output
332 - int current_m_block // current m block to start kernel computation from
333 -) {
334 - // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
335 - // better partitioning with less reductions
336 - int parallel = 1;
337 - if (prob_m > 16 * thread_m_blocks) {
338 - parallel = prob_m / (16 * thread_m_blocks);
339 - prob_m = 16 * thread_m_blocks;
340 - }
341 -
342 - int k_tiles = prob_k / 16 / thread_k_blocks;
343 - int n_tiles = prob_n / 16 / thread_n_blocks;
344 - int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
345 -
346 - if constexpr (!has_act_order && group_blocks != -1) {
347 - if (group_blocks >= thread_k_blocks) {
348 - // Ensure that the number of tiles in each stripe is a multiple of the
349 - // groupsize; this avoids an annoying special case where a stripe starts
350 - // in the middle of group.
351 - iters = (group_blocks / thread_k_blocks) *
352 - ceildiv(iters, (group_blocks / thread_k_blocks));
353 - }
354 - }
355 -
356 - int slice_row = (iters * blockIdx.x) % k_tiles;
357 - int slice_col_par = (iters * blockIdx.x) / k_tiles;
358 - int slice_col = slice_col_par;
359 - int slice_iters; // number of threadblock tiles in the current slice
360 - int slice_count =
361 - 0; // total number of active threadblocks in the current slice
362 - int slice_idx; // index of threadblock in current slice; numbered bottom to
363 - // top
364 -
365 - // We can easily implement parallel problem execution by just remapping
366 - // indices and advancing global pointers
367 - if (slice_col_par >= n_tiles) {
368 - locks += (slice_col_par / n_tiles) * n_tiles;
369 - slice_col = slice_col_par % n_tiles;
370 - sorted_ids += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
371 - }
372 -
373 - // Compute all information about the current slice which is required for
374 - // synchronization.
375 - auto init_slice = [&]() {
376 - slice_iters =
377 - iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
378 - if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
379 - if (slice_iters == 0) return;
380 - if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
381 - slice_count = 1;
382 - slice_idx = 0;
383 - int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
384 - if (col_first <= k_tiles * (slice_col_par + 1)) {
385 - int col_off = col_first - k_tiles * slice_col_par;
386 - slice_count = ceildiv(k_tiles - col_off, iters);
387 - if (col_off > 0) slice_count++;
388 - int delta_first = iters * blockIdx.x - col_first;
389 - if (delta_first < 0 || (col_off == 0 && delta_first == 0))
390 - slice_idx = slice_count - 1;
391 - else {
392 - slice_idx = slice_count - 1 - delta_first / iters;
393 - if (col_off > 0) slice_idx--;
394 - }
395 - }
396 - if (slice_col == n_tiles) {
397 - sorted_ids += 16 * thread_m_blocks;
398 - locks += n_tiles;
399 - slice_col = 0;
400 - }
401 - };
402 - init_slice();
403 -
404 - // A sizes/strides
405 -
406 - // stride of the A matrix in global memory
407 - int a_gl_stride = prob_k / 8;
408 - // stride of an A matrix tile in shared memory
409 - constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
410 - // delta between subsequent A tiles in global memory
411 - constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
412 - // between subsequent accesses within a tile
413 - int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
414 - // between shared memory writes
415 - constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
416 - // between shared memory tile reads
417 - constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
418 - // within a shared memory tile
419 - constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
420 - // overall size of a tile
421 - constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
422 - // number of shared write iterations for a tile
423 - constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
424 -
425 - // B sizes/strides
426 - int b_gl_stride = 16 * prob_n / 32;
427 - constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
428 - int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
429 - int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
430 - constexpr int b_sh_wr_delta = threads;
431 - constexpr int b_sh_rd_delta = threads;
432 - constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
433 - constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
434 -
435 - // Scale sizes/strides without act_order
436 - int s_gl_stride = prob_n / 8;
437 - constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
438 - constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
439 - ? thread_k_blocks / group_blocks
440 - : 1;
441 - constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
442 - int s_gl_rd_delta = s_gl_stride;
443 - // Scale size/strides with act_order
444 - constexpr int tb_k = 16 * thread_k_blocks;
445 - constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
446 - // constexpr int act_s_row_stride = 1;
447 - // int act_s_col_stride = act_s_row_stride * num_groups;
448 - int act_s_col_stride = 1;
449 - int act_s_col_warp_stride = act_s_col_stride * 8;
450 - int tb_n_warps = thread_n_blocks / 4;
451 - int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
452 -
453 - constexpr int sorted_sh_stride = threads;
454 - constexpr int sorted_gl_stride = threads;
455 -
456 - // Global A read index of current thread.
457 - int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
458 - (threadIdx.x % a_gl_rd_delta_o);
459 - a_gl_rd += a_gl_rd_delta_o * slice_row;
460 - // Shared write index of current thread.
461 - int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
462 - (threadIdx.x % a_gl_rd_delta_o);
463 - // Shared read index.
464 - int a_sh_rd =
465 - a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
466 - a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
467 -
468 - int b_gl_rd =
469 - b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
470 - b_gl_rd += b_sh_stride * slice_col;
471 - b_gl_rd += b_gl_rd_delta_o * slice_row;
472 - int b_sh_wr = threadIdx.x;
473 - int b_sh_rd = threadIdx.x;
474 -
475 - // For act_order
476 - constexpr int k_iter_size = tb_k / b_sh_wr_iters;
477 - int slice_k_start = tb_k * slice_row;
478 - int slice_k_finish = slice_k_start + tb_k * slice_iters;
479 - int slice_k_start_shared_fetch = slice_k_start;
480 - int slice_n_offset = act_s_col_tb_stride * slice_col;
481 -
482 - // No act_order
483 - int s_gl_rd;
484 - if constexpr (group_blocks == -1 || group_blocks == 0) {
485 - s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
486 - } else {
487 - s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
488 - s_sh_stride * slice_col + threadIdx.x;
489 - }
490 - int s_sh_wr = threadIdx.x;
491 - bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
492 -
493 - // We use a different scale layout for grouped and column-wise quantization as
494 - // we scale a `half2` tile in column-major layout in the former and in
495 - // row-major in the latter case.
496 - int s_sh_rd;
497 - if constexpr (group_blocks != -1)
498 - s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
499 - (threadIdx.x % 32) / 4;
500 - else
501 - s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
502 - (threadIdx.x % 32) % 4;
503 -
504 - int sh_first_group_id = -1;
505 - int sh_num_groups = -1;
506 - constexpr int sh_max_num_groups = 32;
507 -
508 - int shs_size;
509 - if constexpr (has_act_order)
510 - shs_size = sh_max_num_groups * s_sh_stride + threads;
511 - else
512 - shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
513 -
514 - extern __shared__ int4 sh[];
515 - // Shared memory storage for global fetch pipelines.
516 - int4* sh_a = sh;
517 - int4* sh_b = sh_a + (stages * a_sh_stage);
518 - int4* sh_g_idx = sh_b + (stages * b_sh_stage);
519 - int4* sh_s = sh_g_idx + (stages * g_idx_stage);
520 - int* sh_sorted = (int*)(sh_s + shs_size);
521 -
522 - // Precompute which thread should not read memory in which iterations; this is
523 - // needed if there are more threads than required for a certain tilesize or
524 - // when the batchsize is not a multiple of 16.
525 - bool a_sh_wr_pred[a_sh_wr_iters];
526 - #pragma unroll
527 - for (int i = 0; i < a_sh_wr_iters; i++) {
528 - int a_idx = a_sh_wr_delta * i + a_sh_wr;
529 - int row = a_idx / a_gl_rd_delta_o;
530 - if (row >= prob_m) {
531 - a_sh_wr_pred[i] = false;
532 - } else {
533 - a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
534 - }
535 - }
536 -
537 - // To ensure that writing and reading A tiles to/from shared memory, the
538 - // latter in fragment format, is fully bank conflict free, we need to use a
539 - // rather fancy XOR-based layout. The key here is that neither reads nor
540 - // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
541 - // same shared memory banks. Further, it seems (based on NSight-Compute) that
542 - // each warp must also write a consecutive memory segment?
543 - auto transform_a = [&](int i) {
544 - int row = i / a_gl_rd_delta_o;
545 - return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
546 - };
547 - // Since the computation of this remapping is non-trivial and, due to our main
548 - // loop unrolls, all shared memory accesses are static, we simply precompute
549 - // both transformed reads and writes.
550 - int a_sh_wr_trans[a_sh_wr_iters];
551 - #pragma unroll
552 - for (int i = 0; i < a_sh_wr_iters; i++)
553 - a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
554 - int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
555 - #pragma unroll
556 - for (int i = 0; i < b_sh_wr_iters; i++) {
557 - #pragma unroll
558 - for (int j = 0; j < thread_m_blocks; j++)
559 - a_sh_rd_trans[i][j] =
560 - transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
561 - }
562 -
563 - // Since B-accesses have non-constant stride they have to be computed at
564 - // runtime; we break dependencies between subsequent accesses with a tile by
565 - // maintining multiple pointers (we have enough registers), a tiny
566 - // optimization.
567 - const int4* B_ptr[b_sh_wr_iters];
568 - #pragma unroll
569 - for (int i = 0; i < b_sh_wr_iters; i++)
570 - B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
571 -
572 - // Register storage for double buffer of shared memory reads.
573 - FragA frag_a[2][thread_m_blocks];
574 - I4 frag_b_quant[2];
575 - FragC frag_c[thread_m_blocks][4][2];
576 - FragS frag_s[2][4]; // No act-order
577 - FragS act_frag_s[2][4][4]; // For act-order
578 -
579 - // Zero accumulators.
580 - auto zero_accums = [&]() {
581 - #pragma unroll
582 - for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
583 - reinterpret_cast<float*>(frag_c)[i] = 0;
584 - };
585 -
586 - auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
587 - int last_group_id) {
588 - sh_first_group_id = first_group_id;
589 - sh_num_groups = last_group_id - first_group_id + 1;
590 -
591 - if (sh_num_groups < sh_max_num_groups) {
592 - sh_num_groups = sh_max_num_groups;
593 - }
594 -
595 - if (sh_first_group_id + sh_num_groups > num_groups) {
596 - sh_num_groups = num_groups - sh_first_group_id;
597 - }
598 -
599 - int row_offset = first_group_id * s_gl_stride;
600 -
601 - if (is_async) {
602 - for (int i = 0; i < sh_num_groups; i++) {
603 - if (threadIdx.x < s_sh_stride) {
604 - cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
605 - &scales_ptr[row_offset + (i * s_gl_stride) +
606 - slice_n_offset + threadIdx.x]);
607 - }
608 - }
609 - } else {
610 - for (int i = 0; i < sh_num_groups; i++) {
611 - if (threadIdx.x < s_sh_stride) {
612 - sh_s[(i * s_sh_stride) + threadIdx.x] =
613 - scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
614 - threadIdx.x];
615 - }
616 - }
617 - }
618 - };
619 - // Asynchronously fetch the next A, B and s tile from global to the next
620 - // shared memory pipeline location.
621 - auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
622 - if (pred) {
623 - int4* sh_a_stage = sh_a + a_sh_stage * pipe;
624 - #pragma unroll
625 - for (int i = 0; i < a_sh_wr_iters; i++) {
626 - int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off;
627 - int row = a_idx / a_gl_stride;
628 - int sorted_row =
629 - replicate_input ? sorted_ids[row] / topk : sorted_ids[row];
630 - int new_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride;
631 - if (sorted_row < tot_m * (replicate_input ? 1 : topk) &&
632 - new_idx < a_gl_stride * tot_m * (replicate_input ? 1 : topk)) {
633 - cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[new_idx],
634 - a_sh_wr_pred[i]);
635 - }
636 - }
637 - int4* sh_b_stage = sh_b + b_sh_stage * pipe;
638 - #pragma unroll
639 - for (int i = 0; i < b_sh_wr_iters; i++) {
640 - cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
641 - B_ptr[i] += b_gl_rd_delta_o;
642 - }
643 -
644 - if constexpr (has_act_order) {
645 - // Fetch g_idx thread-block portion
646 - int full_pipe = a_off;
647 - int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
648 - if (cur_k < prob_k && cur_k < slice_k_finish) {
649 - int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
650 -
651 - int4 const* cur_g_idx_stage_ptr =
652 - reinterpret_cast<int4 const*>(&g_idx[cur_k]);
653 -
654 - if (threadIdx.x < g_idx_stage) {
655 - cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
656 - &cur_g_idx_stage_ptr[threadIdx.x]);
657 - }
658 - }
659 - } else {
660 - if constexpr (group_blocks != -1) {
661 - int4* sh_s_stage = sh_s + s_sh_stage * pipe;
662 -
663 - if constexpr (group_blocks >= thread_k_blocks) {
664 - // Only fetch scales if this tile starts a new group
665 - if (pipe % (group_blocks / thread_k_blocks) == 0) {
666 - if (s_sh_wr_pred) {
667 - cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
668 - }
669 - s_gl_rd += s_gl_rd_delta;
670 - }
671 - } else {
672 - for (int i = 0; i < s_tb_groups; i++) {
673 - if (s_sh_wr_pred) {
674 - cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
675 - &scales_ptr[s_gl_rd]);
676 - }
677 - s_gl_rd += s_gl_rd_delta;
678 - }
679 - }
680 - }
681 - }
682 - }
683 - // Insert a fence even when we are winding down the pipeline to ensure that
684 - // waiting is also correct at this point.
685 - cp_async_fence();
686 - };
687 -
688 - // TODO we are currently hitting illegal memory accesses when fetching
689 - // sorted_ids to shared data: fix this
690 - auto fetch_sorted_ids_to_shared = [&]() {
691 - const int mpt = ceildiv(prob_m, threads);
692 - for (int i = 0; i < mpt; i++) {
693 - if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
694 - sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
695 - sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
696 - }
697 - }
698 - };
699 -
700 - // Wait until the next thread tile has been loaded to shared memory.
701 - auto wait_for_stage = [&]() {
702 - // We only have `stages - 2` active fetches since we are double buffering
703 - // and can only issue the next fetch when it is guaranteed that the previous
704 - // shared memory load is fully complete (as it may otherwise be
705 - // overwritten).
706 - cp_async_wait<stages - 2>();
707 - __syncthreads();
708 - };
709 -
710 - // Load the next sub-tile from the current location in the shared memory pipe
711 - // into the current register buffer.
712 - auto fetch_to_registers = [&](int k, int pipe) {
713 - int4* sh_a_stage = sh_a + a_sh_stage * pipe;
714 - #pragma unroll
715 - for (int i = 0; i < thread_m_blocks; i++)
716 - ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
717 - int4* sh_b_stage = sh_b + b_sh_stage * pipe;
718 - frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
719 - &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
720 - };
721 -
722 - bool is_same_group[stages];
723 - int same_group_id[stages];
724 -
725 - auto init_same_group = [&](int pipe) {
726 - int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
727 - int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
728 -
729 - int group_id_1 = sh_g_idx_int_ptr[0];
730 - int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
731 -
732 - is_same_group[pipe] = group_id_1 == group_id_2;
733 - same_group_id[pipe] = group_id_1;
734 - };
735 -
736 - auto fetch_scales_to_registers = [&](int k, int full_pipe) {
737 - int pipe = full_pipe % stages;
738 -
739 - if constexpr (!has_act_order) {
740 - // No act-order case
741 - if constexpr (group_blocks != -1) {
742 - if constexpr (group_blocks >= thread_k_blocks) {
743 - int4* sh_s_stage =
744 - sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
745 - (pipe / (group_blocks / thread_k_blocks)));
746 - reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
747 - } else {
748 - int warp_id = threadIdx.x / 32;
749 - int n_warps = thread_n_blocks / 4;
750 -
751 - int warp_row = warp_id / n_warps;
752 -
753 - int cur_k = warp_row * 16;
754 - cur_k += k_iter_size * (k % b_sh_wr_iters);
755 -
756 - int k_blocks = cur_k / 16;
757 - int cur_group_id = k_blocks / group_blocks;
758 -
759 - int4* sh_s_stage = sh_s + s_sh_stage * pipe;
760 -
761 - reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
762 - sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
763 - }
764 - }
765 -
766 - return;
767 - }
768 -
769 - // Act-order case
770 -
771 - // Determine K of the "current" thread-block
772 - int cur_k = slice_k_start + tb_k * full_pipe;
773 - if (cur_k >= prob_k || cur_k >= slice_k_finish) {
774 - return;
775 - }
776 -
777 - // Reset (to current thread-block) since we read g_idx portion from the
778 - // shared memory
779 - cur_k = 0;
780 -
781 - // Progress to current iteration
782 - cur_k += k_iter_size * (k % b_sh_wr_iters);
783 -
784 - // Determine "position" inside the thread-block (based on warp and
785 - // thread-id)
786 - int warp_id = threadIdx.x / 32;
787 - int n_warps =
788 - thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
789 -
790 - int warp_row = warp_id / n_warps;
791 - int warp_col = warp_id % n_warps;
792 -
793 - cur_k += warp_row * 16;
794 -
795 - int th_id = threadIdx.x % 32;
796 - cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
797 -
798 - int s_col_shift =
799 - /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
800 - (th_id / 4) * act_s_col_stride;
801 -
802 - if (is_same_group[pipe]) {
803 - if (k % 2 == 0) {
804 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
805 - sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
806 - s_col_shift];
807 - } else {
808 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
809 - *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
810 - }
811 -
812 - for (int i = 1; i < 4; i++) {
813 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
814 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
815 - }
816 - return;
817 - }
818 -
819 - int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
820 - int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
821 -
822 - constexpr int k_frag_offsets[4] = {0, 1, 8,
823 - 9}; // Tensor core offsets per thread
824 -
825 - #pragma unroll
826 - for (int i = 0; i < 4; i++) {
827 - int actual_k = cur_k + k_frag_offsets[i];
828 -
829 - int group_id = sh_g_idx_int_ptr[actual_k];
830 - int rel_group_id = group_id - sh_first_group_id;
831 -
832 - *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
833 - sh_s[rel_group_id * s_sh_stride + s_col_shift];
834 - }
835 - };
836 -
837 - // Execute the actual tensor core matmul of a sub-tile.
838 - auto matmul = [&](int k) {
839 - // We have the m dimension as the inner loop in order to encourage overlapping
840 - // dequantization and matmul operations.
841 - #pragma unroll
842 - for (int j = 0; j < 4; j++) {
843 - int b_quant = frag_b_quant[k % 2][j];
844 - int b_quant_shift = b_quant >> 8;
845 -
846 - FragB frag_b0 = dequant(b_quant);
847 -
848 - // Apply scale to frag_b0
849 - if constexpr (has_act_order) {
850 - scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
851 - act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
852 - } else {
853 - if constexpr (group_blocks != -1) {
854 - scale(frag_b0, frag_s[k % 2][j], 0);
855 - }
856 - }
857 -
858 - FragB frag_b1 = dequant(b_quant_shift);
859 -
860 - // Apply scale to frag_b1
861 - if constexpr (has_act_order) {
862 - scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
863 - act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
864 -
865 - } else {
866 - if constexpr (group_blocks != -1) {
867 - scale(frag_b1, frag_s[k % 2][j], 1);
868 - }
869 - }
870 -
871 - #pragma unroll
872 - for (int i = 0; i < thread_m_blocks; i++) {
873 - mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
874 - mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
875 - }
876 - }
877 - };
878 -
879 - // Since we slice across the k dimension of a tile in order to increase the
880 - // number of warps while keeping the n dimension of a tile reasonable, we have
881 - // multiple warps that accumulate their partial sums of the same output
882 - // location; which we have to reduce over in the end. We do in shared memory.
883 - auto thread_block_reduce = [&]() {
884 - constexpr int red_off = threads / b_sh_stride / 2;
885 - if (red_off >= 1) {
886 - int red_idx = threadIdx.x / b_sh_stride;
887 - constexpr int red_sh_stride = b_sh_stride * 4 * 2;
888 - constexpr int red_sh_delta = b_sh_stride;
889 - int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
890 - (threadIdx.x % b_sh_stride);
891 -
892 - // Parallel logarithmic shared memory reduction. We make sure to avoid any
893 - // unnecessary read or write iterations, e.g., for two warps we write only
894 - // once by warp 1 and read only once by warp 0.
895 -
896 - #pragma unroll
897 - for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
898 - #pragma unroll
899 - for (int i = red_off; i > 0; i /= 2) {
900 - if (i <= red_idx && red_idx < 2 * i) {
901 - #pragma unroll
902 - for (int j = 0; j < 4 * 2; j++) {
903 - int red_sh_wr =
904 - red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
905 - if (i < red_off) {
906 - float* c_rd =
907 - reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
908 - float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
909 - #pragma unroll
910 - for (int k = 0; k < 4; k++)
911 - reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
912 - c_rd[k] + c_wr[k];
913 - }
914 - sh[red_sh_wr] =
915 - reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
916 - }
917 - }
918 - __syncthreads();
919 - }
920 - if (red_idx == 0) {
921 - #pragma unroll
922 - for (int i = 0; i < 4 * 2; i++) {
923 - float* c_rd =
924 - reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
925 - #pragma unroll
926 - for (int j = 0; j < 4; j++)
927 - reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
928 - c_rd[j];
929 - }
930 - }
931 - __syncthreads();
932 - }
933 - }
934 - };
935 -
936 - // Since multiple threadblocks may process parts of the same column slice, we
937 - // finally have to globally reduce over the results. As the striped
938 - // partitioning minimizes the number of such reductions and our outputs are
939 - // usually rather small, we perform this reduction serially in L2 cache.
940 - auto global_reduce = [&](bool first = false, bool last = false) {
941 - // We are very careful here to reduce directly in the output buffer to
942 - // maximize L2 cache utilization in this step. To do this, we write out
943 - // results in FP16 (but still reduce with FP32 compute).
944 - constexpr int active_threads = 32 * thread_n_blocks / 4;
945 - if (threadIdx.x < active_threads) {
946 - int c_gl_stride = prob_n / 8;
947 - int c_gl_wr_delta_o = 8 * c_gl_stride;
948 - int c_gl_wr_delta_i = 4 * (active_threads / 32);
949 - int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
950 - 4 * (threadIdx.x / 32) + threadIdx.x % 4;
951 - c_gl_wr += (2 * thread_n_blocks) * slice_col;
952 - constexpr int c_sh_wr_delta = active_threads;
953 - int c_sh_wr = threadIdx.x;
954 -
955 - int row = (threadIdx.x % 32) / 4;
956 -
957 - if (!first) {
958 - // Interestingly, doing direct global accesses here really seems to mess up
959 - // the compiler and lead to slowdowns, hence we also use async-copies even
960 - // though these fetches are not actually asynchronous.
961 - #pragma unroll
962 - for (int i = 0; i < thread_m_blocks * 4; i++) {
963 - int c_idx =
964 - c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
965 - int sorted_row = sorted_ids[c_idx / c_gl_stride];
966 - int new_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride;
967 - cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i], &C[new_idx],
968 - sorted_row < tot_m * topk &&
969 - (8 * (i / 2) + row < prob_m &&
970 - (i < (thread_m_blocks - 1) * 4 ||
971 - sorted_ids[8 * (i / 2) + row] < tot_m * topk)));
972 - }
973 - cp_async_fence();
974 - cp_async_wait<0>();
975 - }
976 -
977 - #pragma unroll
978 - for (int i = 0; i < thread_m_blocks * 4; i++) {
979 - if (8 * (i / 2) + row < prob_m &&
980 - (i < (thread_m_blocks - 1) * 4 ||
981 - sorted_ids[8 * (i / 2) + row] < tot_m * topk)) {
982 - if (!first) {
983 - int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
984 - #pragma unroll
985 - for (int j = 0; j < 2 * 4; j++) {
986 - reinterpret_cast<float*>(
987 - &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
988 - __half2float(reinterpret_cast<__half*>(&c_red)[j]);
989 - }
990 - }
991 - if (!last) {
992 - int4 c;
993 - #pragma unroll
994 - for (int j = 0; j < 2 * 4; j++) {
995 - reinterpret_cast<__half*>(&c)[j] =
996 - __float2half(reinterpret_cast<float*>(
997 - &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
998 - }
999 - int c_idx =
1000 - c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
1001 - int row = sorted_ids[c_idx / c_gl_stride];
1002 - if (row < tot_m * topk) {
1003 - int new_idx = row * c_gl_stride + c_idx % c_gl_stride;
1004 - C[new_idx] = c;
1005 - }
1006 - }
1007 - }
1008 - }
1009 - }
1010 - };
1011 -
1012 - // Write out the reduce final result in the correct layout. We only actually
1013 - // reshuffle matrix fragments in this step, the reduction above is performed
1014 - // in fragment layout.
1015 - auto write_result = [&]() {
1016 - int c_gl_stride = prob_n / 8;
1017 - constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
1018 - int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
1019 - constexpr int c_sh_rd_delta =
1020 - c_sh_stride * (threads / (2 * thread_n_blocks));
1021 -
1022 - int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1023 - (threadIdx.x % (2 * thread_n_blocks));
1024 - c_gl_wr += (2 * thread_n_blocks) * slice_col;
1025 - int c_sh_wr =
1026 - (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
1027 - c_sh_wr += 32 * (threadIdx.x / 32);
1028 - int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1029 - (threadIdx.x % (2 * thread_n_blocks));
1030 -
1031 - int c_gl_wr_end = c_gl_stride * prob_m;
1032 -
1033 - // We first reorder in shared memory to guarantee the most efficient final
1034 - // global write patterns
1035 - auto write = [&](int idx, float c0, float c1, FragS& s) {
1036 - half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1037 -
1038 - // For per-column quantization we finally apply the scale here
1039 - if constexpr (!has_act_order && group_blocks == -1) {
1040 - res = __hmul2(res, s[0]);
1041 - }
1042 -
1043 - ((half2*)sh)[idx] = res;
1044 - };
1045 - if (threadIdx.x / 32 < thread_n_blocks / 4) {
1046 - #pragma unroll
1047 - for (int i = 0; i < thread_m_blocks; i++) {
1048 - #pragma unroll
1049 - for (int j = 0; j < 4; j++) {
1050 - int wr = c_sh_wr + 8 * j;
1051 - write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
1052 - frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
1053 - write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
1054 - frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
1055 - write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
1056 - frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
1057 - write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
1058 - frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
1059 - }
1060 - c_sh_wr += 16 * (4 * c_sh_stride);
1061 - }
1062 - }
1063 - __syncthreads();
1064 -
1065 - #pragma unroll
1066 - for (int i = 0;
1067 - i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1068 - i++) {
1069 - if (c_gl_wr < c_gl_wr_end) {
1070 - int row = sorted_ids[c_gl_wr / c_gl_stride];
1071 - if (row < tot_m * topk) {
1072 - int off = row * c_gl_stride + c_gl_wr % c_gl_stride;
1073 - if (!apply_weights) {
1074 - C[off] = sh[c_sh_rd];
1075 - } else {
1076 - __half* ctrg = reinterpret_cast<__half*>(&C[off]);
1077 - __half* csrc = reinterpret_cast<__half*>(&sh[c_sh_rd]);
1078 - for (int j = 0; j < 8; ++j) {
1079 - ctrg[j] = __float2half(topk_weights[row] * __half2float(csrc[j]));
1080 - }
1081 - }
1082 - c_gl_wr += c_gl_wr_delta;
1083 - c_sh_rd += c_sh_rd_delta;
1084 - }
1085 - }
1086 - }
1087 - };
1088 -
1089 - // Start global fetch and register load pipelines.
1090 - auto start_pipes = [&]() {
1091 - // TODO re-enable after fixing this function
1092 - // fetch_sorted_ids_to_shared();
1093 - __syncthreads();
1094 -
1095 - #pragma unroll
1096 - for (int i = 0; i < stages - 1; i++) {
1097 - if (has_act_order && i == 0) {
1098 - int last_g_idx = slice_k_start + stages * tb_k * 2;
1099 - if (last_g_idx >= prob_k) {
1100 - last_g_idx = prob_k - 1;
1101 - }
1102 - fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1103 - }
1104 - fetch_to_shared(i, i, i < slice_iters);
1105 - }
1106 -
1107 - zero_accums();
1108 - wait_for_stage();
1109 - init_same_group(0);
1110 - fetch_to_registers(0, 0);
1111 - fetch_scales_to_registers(0, 0);
1112 - a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1113 - slice_k_start_shared_fetch += tb_k * (stages - 1);
1114 - };
1115 - if (slice_iters) {
1116 - start_pipes();
1117 - }
1118 -
1119 - // Main loop.
1120 - while (slice_iters) {
1121 - // We unroll over both the global fetch and the register load pipeline to
1122 - // ensure all shared memory accesses are static. Note that both pipelines
1123 - // have even length meaning that the next iteration will always start at
1124 - // index 0.
1125 - #pragma unroll
1126 - for (int pipe = 0; pipe < stages;) {
1127 - #pragma unroll
1128 - for (int k = 0; k < b_sh_wr_iters; k++) {
1129 - fetch_to_registers(k + 1, pipe % stages);
1130 - fetch_scales_to_registers(k + 1, pipe);
1131 - if (k == b_sh_wr_iters - 2) {
1132 - fetch_to_shared((pipe + stages - 1) % stages, pipe,
1133 - slice_iters >= stages);
1134 - pipe++;
1135 - wait_for_stage();
1136 - init_same_group(pipe % stages);
1137 - }
1138 - matmul(k);
1139 - }
1140 - slice_iters--;
1141 - if (slice_iters == 0) {
1142 - break;
1143 - }
1144 - }
1145 -
1146 - a_gl_rd += a_gl_rd_delta_o * stages;
1147 - slice_k_start += tb_k * stages;
1148 - slice_k_start_shared_fetch += tb_k * stages;
1149 -
1150 - if constexpr (has_act_order) {
1151 - int first_group_id = g_idx[slice_k_start];
1152 - int last_g_idx = slice_k_start + stages * tb_k * 2;
1153 - if (last_g_idx >= prob_k) {
1154 - last_g_idx = prob_k - 1;
1155 - }
1156 - int last_group_id = g_idx[last_g_idx];
1157 - if (last_group_id >= sh_first_group_id + sh_num_groups) {
1158 - fetch_scales_to_shared(false, first_group_id, last_group_id);
1159 - __syncthreads();
1160 - }
1161 - }
1162 -
1163 - // Process results and, if necessary, proceed to the next column slice.
1164 - // While this pattern may not be the most readable, other ways of writing
1165 - // the loop seemed to noticeably worse performance after compilation.
1166 - if (slice_iters == 0) {
1167 - cp_async_wait<0>();
1168 - bool last = slice_idx == slice_count - 1;
1169 - // For per-column scales, we only fetch them here in the final step before
1170 - // write-out
1171 - if constexpr (!has_act_order && group_blocks == -1) {
1172 - if (last) {
1173 - if (s_sh_wr_pred) {
1174 - cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1175 - }
1176 - cp_async_fence();
1177 - }
1178 - }
1179 -
1180 - thread_block_reduce();
1181 - if constexpr (!has_act_order && group_blocks == -1) {
1182 - if (last) {
1183 - cp_async_wait<0>();
1184 - __syncthreads();
1185 - if (threadIdx.x / 32 < thread_n_blocks / 4) {
1186 - reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1187 - reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1188 - }
1189 - }
1190 - }
1191 - if (slice_count > 1) { // only globally reduce if there is more than one
1192 - // block in a slice
1193 - barrier_acquire(&locks[slice_col], slice_idx);
1194 - global_reduce(slice_idx == 0, last);
1195 - barrier_release(&locks[slice_col], last);
1196 - }
1197 - if (last) // only the last block in a slice actually writes the result
1198 - write_result();
1199 - slice_row = 0;
1200 - slice_col_par++;
1201 - slice_col++;
1202 - init_slice();
1203 - if (slice_iters) {
1204 - a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1205 - (threadIdx.x % a_gl_rd_delta_o);
1206 - #pragma unroll
1207 - for (int i = 0; i < b_sh_wr_iters; i++)
1208 - B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1209 - if (slice_col == 0) {
1210 - #pragma unroll
1211 - for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1212 - }
1213 -
1214 - // Update slice k/n for scales loading
1215 - if constexpr (has_act_order) {
1216 - slice_k_start = tb_k * slice_row;
1217 - slice_k_finish = slice_k_start + tb_k * slice_iters;
1218 - slice_k_start_shared_fetch = slice_k_start;
1219 - slice_n_offset = act_s_col_tb_stride * slice_col;
1220 -
1221 - } else {
1222 - s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1223 - }
1224 - start_pipes();
1225 - }
1226 - }
1227 - }
1228 -}
1229 -
1230 -template <const int threads, // number of threads in a threadblock
1231 - const int thread_m_blocks, // number of 16x16 blocks in the m
1232 - // dimension (batchsize) of the
1233 - // threadblock
1234 - const int thread_n_blocks, // same for n dimension (output)
1235 - const int thread_k_blocks, // same for k dimension (reduction)
1236 - const int stages, // number of stages for the async global->shared
1237 - // fetch pipeline
1238 - const bool has_act_order, // whether act_order is enabled
1239 - const int group_blocks = -1 // number of consecutive 16x16 blocks
1240 - // with a separate quantization scale
1241 - >
1242 -__global__ void MarlinMoE(
1243 - const int4* __restrict__ A, // fp16 input matrix of shape mxk
1244 - const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1245 - int4* __restrict__ C, // fp16 output buffer of shape mxn
1246 - const int* __restrict__ sorted_ids_base, // int32 sorted ids of experts
1247 - const float* __restrict__ topk_weights, // float topk weights
1248 - const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1249 - // (k/groupsize)xn
1250 - const int* __restrict__ g_idx, // int32 group indices of shape k
1251 - const int* __restrict__ expert_offsets,
1252 - int num_groups, // number of scale groups per output channel
1253 - int expert_idx, // idx of current expert
1254 - int num_experts, // number of experts
1255 - int topk, // topk parameter of moe
1256 - int prob_m, // batch dimension m
1257 - int prob_n, // output dimension n
1258 - int prob_k, // reduction dimension k
1259 - int tot_m, // total number of rows in A and C
1260 - int* locks, // extra global storage for barrier synchronization
1261 - bool replicate_input, // do we use the same input for each expert?
1262 - bool apply_weights, // apply weights to output
1263 - int current_m_block, // current m block to start kernel computation from
1264 - int max_par // maximum parallelism
1265 -) {
1266 - int m_block_ctr = current_m_block;
1267 -
1268 - const int* sorted_ids_expert =
1269 - sorted_ids_base + expert_offsets[expert_idx] + m_block_ctr * 4 * max_par;
1270 - int tot_its = expert_offsets[expert_idx + 1] - expert_offsets[expert_idx];
1271 - if (tot_its == 0) {
1272 - return;
1273 - }
1274 - int tot_m_blocks = ceildiv(tot_its, 16);
1275 - int pad = 16 * tot_m_blocks - tot_its;
1276 -
1277 - if (m_block_ctr >= tot_m_blocks) {
1278 - return;
1279 - }
1280 -
1281 - int max_block = tot_m_blocks - m_block_ctr;
1282 - prob_m = tot_its - 16 * m_block_ctr;
1283 -
1284 - int par = 1;
1285 - if (max_block > 4) {
1286 - // Note that parallel > 1 currently only works for inputs without any
1287 - // padding
1288 - par = (16 * max_block - pad) / 64;
1289 - par = min((16 * max_block - pad) / 64, max_par);
1290 - prob_m = 64 * par;
1291 - m_block_ctr += 4 * (par - 1);
1292 - max_block = 4;
1293 - }
1294 -
1295 - if (max_block == 1) {
1296 - MarlinMoESingle<threads, 1, thread_n_blocks, thread_k_blocks, stages,
1297 - has_act_order, group_blocks>(
1298 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1299 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1300 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1301 - current_m_block);
1302 - } else if (max_block == 2) {
1303 - MarlinMoESingle<threads, 2, thread_n_blocks, thread_k_blocks, stages,
1304 - has_act_order, group_blocks>(
1305 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1306 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1307 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1308 - current_m_block);
1309 - } else if (max_block == 3) {
1310 - MarlinMoESingle<threads, 3, thread_n_blocks, thread_k_blocks, stages,
1311 - has_act_order, group_blocks>(
1312 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1313 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1314 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1315 - current_m_block);
1316 - } else {
1317 - MarlinMoESingle<threads, 4, thread_n_blocks, thread_k_blocks, stages,
1318 - has_act_order, group_blocks>(
1319 - A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1320 - expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1321 - prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1322 - current_m_block);
1323 - }
1324 -}
1325 -
1326 -#else
1327 -
1328 -__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
1329 - int const* __restrict__ perm_int_ptr,
1330 - int4* __restrict__ out_int4_ptr, int size_m,
1331 - int size_k, int block_rows) {
1332 - // Marlin is not implemented yet for SM < 8.0
1333 - assert(false);
1334 - return;
1335 -}
1336 -
1337 -__global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
1338 - int* __restrict__ expert_offsets,
1339 - int topk_length, int block_size) {
1340 - // Marlin is not implemented yet for SM < 8.0
1341 - assert(false);
1342 - return;
1343 -}
1344 -
1345 -template <const int threads, // number of threads in a threadblock
1346 - const int thread_m_blocks, // number of 16x16 blocks in the m
1347 - // dimension (batchsize) of the
1348 - // threadblock
1349 - const int thread_n_blocks, // same for n dimension (output)
1350 - const int thread_k_blocks, // same for k dimension (reduction)
1351 - const int stages, // number of stages for the async global->shared
1352 - // fetch pipeline
1353 - const bool has_act_order, // whether act_order is enabled
1354 - const int group_blocks = -1 // number of consecutive 16x16 blocks
1355 - // with a separate quantization scale
1356 - >
1357 -__global__ void MarlinMoE(
1358 - const int4* __restrict__ A, // fp16 input matrix of shape mxk
1359 - const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1360 - int4* __restrict__ C, // fp16 output buffer of shape mxn
1361 - const int* __restrict__ sorted_ids, // int32 sorted ids of experts
1362 - const float* __restrict__ topk_weights, // float topk weights
1363 - const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1364 - // (k/groupsize)xn
1365 - const int* __restrict__ g_idx, // int32 group indices of shape k
1366 - const int* __restrict__ expert_offsets,
1367 - int num_groups, // number of scale groups per output channel
1368 - int expert_idx, // idx of current expert
1369 - int num_experts, // number of experts
1370 - int topk, // topk parameter of moe
1371 - int prob_m, // batch dimension m
1372 - int prob_n, // output dimension n
1373 - int prob_k, // reduction dimension k
1374 - int tot_m, // total number of rows in A and C
1375 - int* locks, // extra global storage for barrier synchronization
1376 - bool replicate_input, // do we use the same input for each expert?
1377 - bool apply_weights, // apply weights to output
1378 - int current_m_block, // current m block to start kernel computation from
1379 - int max_par // maximum parallelism
1380 -) {
1381 - // Marlin is not implemented yet for SM < 8.0
1382 - assert(false);
1383 - return;
1384 -}
1385 -
1386 -#endif
1387 -
1388 -// 8 warps are a good choice since every SM has 4 schedulers and having more
1389 -// than 1 warp per schedule allows some more latency hiding. At the same time,
1390 -// we want relatively few warps to have many registers per warp and small tiles.
1391 -const int USER_THREADS =
1392 - 256; // Note: This is only used with user-provided thread_k/n
1393 -const int STAGES = 4; // 4 pipeline stages fit into shared memory
1394 -// const int SHARED_MEM =
1395 -// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1396 -
1397 -static constexpr int min_thread_n = 64;
1398 -static constexpr int min_thread_k = 64;
1399 -
1400 -#define __CALL_IF_MOE(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1401 - HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1402 - else if (thread_m_blocks == THREAD_M_BLOCKS && \
1403 - thread_n_blocks == THREAD_N_BLOCKS && \
1404 - thread_k_blocks == THREAD_K_BLOCKS && \
1405 - has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1406 - num_threads == NUM_THREADS) { \
1407 - cudaFuncSetAttribute( \
1408 - MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1409 - THREAD_K_BLOCKS, STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1410 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1411 - MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1412 - STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1413 - <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1414 - A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1415 - g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1416 - num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1417 - replicate_input, apply_weights, m_block, max_par); \
1418 - }
1419 -
1420 -typedef struct {
1421 - int thread_k;
1422 - int thread_n;
1423 - int num_threads;
1424 -} thread_config_t;
1425 -
1426 -thread_config_t small_batch_thread_configs[] = {
1427 - // Ordered by priority
1428 -
1429 - // thread_k, thread_n, num_threads
1430 - {128, 128, 256}, // Default
1431 - {128, 64, 128}, // Reduce N 2X, same K
1432 - {64, 256, 256}, // Reduce K 2X, increase N 2X
1433 - {64, 128, 128}, // Reduce K 2X, same N
1434 -};
1435 -
1436 -thread_config_t large_batch_thread_configs[] = {
1437 - // Ordered by priority
1438 -
1439 - // thread_k, thread_n, num_threads
1440 - {64, 256, 256}, // Default
1441 - {128, 128, 256}, // Reduce N 2X, increase K 2X
1442 - {64, 128, 128}, // Reduce N 2X, same K
1443 - {128, 64, 128}, // Reduce N 4X, increase K 2X
1444 -};
1445 -
1446 -bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
1447 - int prob_k) {
1448 - // Sanity
1449 - if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1450 - th_config.num_threads == -1) {
1451 - return false;
1452 - }
1453 -
1454 - // Verify K/N are divisible by thread K/N
1455 - if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) {
1456 - return false;
1457 - }
1458 -
1459 - // thread_k can be only 128 or 64 (because it must be less than groupsize
1460 - // which is 128)
1461 - if (th_config.thread_k != 128 && th_config.thread_k != 64) {
1462 - return false;
1463 - }
1464 -
1465 - // Verify min for thread K/N
1466 - if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) {
1467 - return false;
1468 - }
1469 -
1470 - // num_threads must be at least 128 (= 4 warps)
1471 - if (th_config.num_threads < 128) {
1472 - return false;
1473 - }
1474 -
1475 - return true;
1476 -}
1477 -
1478 -thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1479 - if (prob_m <= 16) {
1480 - for (auto th_config : small_batch_thread_configs) {
1481 - if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1482 - return th_config;
1483 - }
1484 - }
1485 -
1486 - } else {
1487 - for (auto th_config : large_batch_thread_configs) {
1488 - if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1489 - return th_config;
1490 - }
1491 - }
1492 - }
1493 -
1494 - return thread_config_t{-1, -1, -1};
1495 -}
1496 -
1497 -#define CALL_IF_MOE(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1498 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1499 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1500 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1501 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1502 - \
1503 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1504 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1505 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1506 - __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1507 - \
1508 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1509 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1510 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1511 - __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1512 - \
1513 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1514 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1515 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1516 - __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1517 - \
1518 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1519 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1520 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1521 - __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1522 -
1523 -void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1524 - const void* sorted_ids, const void* topk_weights,
1525 - const void* topk_ids, const void* s, const void* g_idx,
1526 - const void* perm, void* a_tmp, void* expert_offsets,
1527 - int prob_m, int prob_n, int prob_k, void* workspace,
1528 - bool has_act_order, bool is_k_full, int num_groups,
1529 - int group_size, int num_experts, int topk,
1530 - int moe_block_size, int dev, cudaStream_t stream,
1531 - int thread_k, int thread_n, int sms, int max_par,
1532 - bool replicate_input, bool apply_weights) {
1533 - TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1534 - ", ", prob_n, ", ", prob_k, "]");
1535 -
1536 - if (sms == -1) {
1537 - cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1538 - }
1539 -
1540 - // Set thread config
1541 - thread_config_t th_config;
1542 - if (thread_k != -1 && thread_n != -1) {
1543 - // User-defined config
1544 - th_config = thread_config_t{thread_k, thread_n, USER_THREADS};
1545 - } else {
1546 - // Auto config
1547 - th_config = determine_thread_config(prob_m, prob_n, prob_k);
1548 - }
1549 -
1550 - TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1551 - "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1552 - ", thread_n = " + str(th_config.thread_n) +
1553 - ", num_threads = " + str(th_config.num_threads) +
1554 - " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1555 - str(prob_n) + "]");
1556 -
1557 - int num_threads = th_config.num_threads;
1558 - thread_k = th_config.thread_k;
1559 - thread_n = th_config.thread_n;
1560 -
1561 - int thread_k_blocks = thread_k / 16;
1562 - int thread_n_blocks = thread_n / 16;
1563 -
1564 - int blocks = sms;
1565 -
1566 - TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
1567 - " is not divisible by thread_n = ", thread_n);
1568 - TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
1569 - " is not divisible by thread_k = ", thread_k);
1570 -
1571 - int group_blocks = 0;
1572 - if (has_act_order) {
1573 - if (is_k_full) {
1574 - TORCH_CHECK(group_size != -1);
1575 - group_blocks = group_size / 16;
1576 - TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1577 - " is not divisible by group_blocks = ", group_blocks);
1578 - } else {
1579 - TORCH_CHECK(group_size == 0);
1580 - group_blocks = 0;
1581 - }
1582 -
1583 - } else {
1584 - if (group_size == -1) {
1585 - group_blocks = -1;
1586 - } else {
1587 - group_blocks = group_size / 16;
1588 - TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1589 - " is not divisible by group_blocks = ", group_blocks);
1590 - }
1591 - }
1592 -
1593 - int max_shared_mem = 0;
1594 - cudaDeviceGetAttribute(&max_shared_mem,
1595 - cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1596 - TORCH_CHECK(max_shared_mem > 0);
1597 -
1598 - int tot_m = prob_m;
1599 -
1600 - const int* topk_ids_ptr = (const int*)topk_ids;
1601 - int* expert_offsets_ptr = (int*)expert_offsets;
1602 - compute_expert_offsets<<<1, num_experts, 0, stream>>>(
1603 - topk_ids_ptr, expert_offsets_ptr, tot_m * topk, moe_block_size);
1604 -
1605 - bool do_permute_a = has_act_order;
1606 -
1607 - // If we have a full K, then we can run the non-act-order version of Marlin
1608 - // (since the weight rows are reordered by increasing group ids, and by
1609 - // having a full K, we have full original groups)
1610 - if (is_k_full) {
1611 - has_act_order = false;
1612 - }
1613 -
1614 - for (int expert_idx = 0; expert_idx < num_experts; ++expert_idx) {
1615 - const int4* A_ptr = (const int4*)A;
1616 - int4* a_tmp_ptr = (int4*)a_tmp;
1617 - const int4* B_ptr = (const int4*)B + (prob_n * prob_k / 32) * expert_idx;
1618 - int4* C_ptr = (int4*)C;
1619 - const float* topk_weights_ptr = (const float*)topk_weights;
1620 - const int* sorted_ids_ptr = (const int*)sorted_ids;
1621 - const int4* s_ptr =
1622 - (const int4*)s +
1623 - (((group_size == -1 || group_size == 0) ? 1 : prob_k / group_size) *
1624 - prob_n / 8) *
1625 - expert_idx;
1626 - const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
1627 - const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
1628 - int* locks = (int*)workspace;
1629 -
1630 - if (do_permute_a) {
1631 - // Permute A columns
1632 - int topk_rows = replicate_input ? tot_m : tot_m * topk;
1633 - int block_rows = ceildiv(topk_rows, blocks);
1634 - permute_cols_kernel<<<blocks, num_threads, 0, stream>>>(
1635 - A_ptr, perm_ptr, a_tmp_ptr, topk_rows, prob_k, block_rows);
1636 - A_ptr = a_tmp_ptr;
1637 - }
1638 -
1639 - int max_m_blocks = ceildiv(tot_m, 16);
1640 - for (int m_block = 0; m_block < max_m_blocks; m_block += 16) {
1641 - // Define kernel configurations
1642 -
1643 - // make it max possible value
1644 - int thread_m_blocks = 4;
1645 -
1646 - if (false) {
1647 - }
1648 - CALL_IF_MOE(16, 4, 256)
1649 - CALL_IF_MOE(8, 8, 256)
1650 - CALL_IF_MOE(8, 4, 128)
1651 - CALL_IF_MOE(4, 8, 128)
1652 - else {
1653 - TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1654 - str(prob_n) + ", " + str(prob_k) + "]" +
1655 - ", has_act_order = " + str(has_act_order) +
1656 - ", num_groups = " + str(num_groups) +
1657 - ", group_size = " + str(group_size) +
1658 - ", thread_m_blocks = " + str(thread_m_blocks) +
1659 - ", thread_n_blocks = " + str(thread_n_blocks) +
1660 - ", thread_k_blocks = " + str(thread_k_blocks));
1661 - }
1662 - }
1663 - }
1664 -}
1665 -
1666 -} // namespace marlin_moe
1667 -
1668 -torch::Tensor marlin_gemm_moe(
1669 - const torch::Tensor& a, const torch::Tensor& b_q_weights,
1670 - const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
1671 - const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
1672 - const torch::Tensor& g_idx, const torch::Tensor& perm,
1673 - torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
1674 - bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
1675 - bool replicate_input, bool apply_weights) {
1676 - int max_par = 4;
1677 -
1678 - int dev = a.get_device();
1679 -
1680 - auto options_dtype =
1681 - torch::TensorOptions().dtype(a.dtype()).device(a.device());
1682 - auto options_int =
1683 - torch::TensorOptions().dtype(torch::kInt).device(a.device());
1684 - torch::Tensor c = torch::zeros({size_m, topk, size_n}, options_dtype);
1685 - torch::Tensor a_tmp =
1686 - replicate_input ? torch::zeros({size_m, size_k}, options_dtype)
1687 - : torch::zeros({size_m, topk, size_k}, options_dtype);
1688 - torch::Tensor expert_offsets = torch::empty({num_experts + 1}, options_int);
1689 -
1690 - // thread_k: `k` size of a thread_tile in `weights` (can usually be left as
1691 - // auto -1)
1692 - int thread_k = -1;
1693 - // thread_n: `n` size of a thread_tile in `weights` (can usually be left as
1694 - // auto -1)
1695 - int thread_n = -1;
1696 - // sms: number of SMs to use for the kernel (can usually be left as auto -1)
1697 - int sms = -1;
1698 -
1699 - // Detect groupsize and act_order
1700 - int num_groups = -1;
1701 - int group_size = -1;
1702 - bool has_act_order = g_idx.size(1) != 0;
1703 -
1704 - int b_rank = b_scales.sizes().size();
1705 - TORCH_CHECK(b_rank == 3, "b_scales rank = ", b_rank, " is not 3");
1706 - TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2),
1707 - " is not size_n = ", size_n);
1708 - num_groups = b_scales.size(1);
1709 -
1710 - if (has_act_order) {
1711 - if (is_k_full) {
1712 - TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
1713 - TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
1714 - ", is not divisible by num_groups = ", num_groups);
1715 - group_size = size_k / num_groups;
1716 - } else {
1717 - group_size = 0;
1718 - }
1719 -
1720 - } else {
1721 - if (num_groups > 1) {
1722 - TORCH_CHECK(
1723 - size_k % num_groups == 0, "size_k = ", size_k,
1724 - ", is not divisible by b_scales.size(0) = ", b_scales.size(0));
1725 - group_size = size_k / num_groups;
1726 - } else {
1727 - group_size = -1;
1728 - }
1729 - }
1730 -
1731 - marlin_moe::marlin_mm_moe_f16i4(
1732 - a.data_ptr(), b_q_weights.data_ptr(), c.data_ptr(), sorted_ids.data_ptr(),
1733 - topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
1734 - g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
1735 - expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
1736 - has_act_order, is_k_full, num_groups, group_size, num_experts, topk,
1737 - moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1738 - thread_n, sms, max_par, replicate_input, apply_weights);
1739 - return c;
1740 -}
\ No newline at end of file
csrc/moe/marlin_moe_ops.h deleted
-12
@@ -1,12 +0,0 @@
1 -#pragma once
2 -
3 -#include <torch/all.h>
4 -
5 -torch::Tensor marlin_gemm_moe(
6 - const torch::Tensor& a, const torch::Tensor& b_q_weights,
7 - const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
8 - const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
9 - const torch::Tensor& g_idx, const torch::Tensor& perm,
10 - torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
11 - bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
12 - bool replicate_input, bool apply_weights);
\ No newline at end of file
csrc/moe/torch_bindings.cpp
-9
@@ -1,6 +1,5 @@
1 #include "core/registration.h"
2 #include "moe_ops.h"
3 -#include "marlin_moe_ops.h"
3
4 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5 // Apply topk softmax to the gating outputs.
@@ -8,14 +7,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
7 "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
8 "token_expert_indices, Tensor gating_output) -> ()");
9 m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
11 - m.def(
12 - "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
13 - "Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
14 - "g_idx, Tensor! perm, Tensor! workspace, int size_m, int size_n, int "
15 - "size_k, bool is_k_full, int num_experts, int topk, int moe_block_size, "
16 - "bool replicate_input, bool apply_weights) -> Tensor");
17 -
18 - m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
10 }
11
12 REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
tests/weight_loading/models.txt
-2
@@ -13,7 +13,5 @@ compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
13 compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
14 compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15 compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16 -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
17 -compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
16 awq, casperhansen/mixtral-instruct-awq, main
17 awq_marlin, casperhansen/mixtral-instruct-awq, main
vllm/_custom_ops.py
-14
@@ -300,20 +300,6 @@ def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int,
300 return torch.ops._C.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits)
301
302
303 -def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
304 - size_k: int, size_n: int,
305 - num_bits: int) -> torch.Tensor:
306 - num_experts = b_q_weight.shape[0]
307 - assert size_k % 16 == 0
308 - output = torch.empty((num_experts, size_k // 16, size_n * 2),
309 - device=b_q_weight.device,
310 - dtype=b_q_weight.dtype)
311 - for e in range(num_experts):
312 - output[e] = torch.ops._C.gptq_marlin_repack(b_q_weight[e], perm[e],
313 - size_k, size_n, num_bits)
314 - return output
315 -
316 -
303 def gptq_marlin_gemm(a: torch.Tensor,
304 b_q_weight: torch.Tensor,
305 b_scales: torch.Tensor,
vllm/model_executor/layers/fused_moe/__init__.py
+8 -6
@@ -1,17 +1,19 @@
1 -from vllm.model_executor.layers.fused_moe.layer import (
2 - FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
1 +from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
2 + FusedMoEMethodBase)
3 from vllm.triton_utils import HAS_TRITON
4
5 -__all__ = ["FusedMoE", "FusedMoEMethodBase", "FusedMoeWeightScaleSupported"]
5 +__all__ = [
6 + "FusedMoE",
7 + "FusedMoEMethodBase",
8 +]
9
10 if HAS_TRITON:
11
12 from vllm.model_executor.layers.fused_moe.fused_moe import (
10 - fused_experts, fused_marlin_moe, fused_moe, fused_topk,
11 - get_config_file_name, grouped_topk)
13 + fused_experts, fused_moe, fused_topk, get_config_file_name,
14 + grouped_topk)
15
16 __all__ += [
14 - "fused_marlin_moe",
17 "fused_moe",
18 "fused_topk",
19 "fused_experts",
vllm/model_executor/layers/fused_moe/fused_moe.py
+18 -116
@@ -323,16 +323,21 @@ def get_moe_configs(E: int, N: int,
323 return None
324
325
326 -def get_default_config(M: int, E: int, N: int, K: int, topk: int,
327 - dtype: Optional[str],
328 - is_marlin: bool) -> Dict[str, int]:
326 +def get_default_config(
327 + M: int,
328 + E: int,
329 + N: int,
330 + K: int,
331 + topk: int,
332 + dtype: Optional[str],
333 +) -> Dict[str, int]:
334 config = {
335 'BLOCK_SIZE_M': 64,
336 'BLOCK_SIZE_N': 64,
337 'BLOCK_SIZE_K': 32,
338 'GROUP_SIZE_M': 8
339 }
335 - if M <= E or (is_marlin and M <= 32):
340 + if M <= E:
341 config = {
342 'BLOCK_SIZE_M': 16,
343 'BLOCK_SIZE_N': 32,
@@ -342,14 +347,14 @@ def get_default_config(M: int, E: int, N: int, K: int, topk: int,
347 return config
348
349
345 -def try_get_optimal_moe_config(w1_shape: Tuple[int, ...],
346 - w2_shape: Tuple[int, ...],
347 - top_k: int,
348 - dtype: Optional[str],
349 - M: int,
350 - override_config: Optional[Dict[str,
351 - Any]] = None,
352 - is_marlin: bool = False):
350 +def try_get_optimal_moe_config(
351 + w1_shape: Tuple[int, ...],
352 + w2_shape: Tuple[int, ...],
353 + top_k: int,
354 + dtype: Optional[str],
355 + M: int,
356 + override_config: Optional[Dict[str, Any]] = None,
357 +):
358 if override_config:
359 config = override_config
360 else:
@@ -363,8 +368,7 @@ def try_get_optimal_moe_config(w1_shape: Tuple[int, ...],
368 config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
369 else:
370 # Else use the default config
366 - config = get_default_config(M, E, N, w1_shape[2], top_k, dtype,
367 - is_marlin)
371 + config = get_default_config(M, E, N, w1_shape[2], top_k, dtype)
372 return config
373
374
@@ -437,108 +441,6 @@ def grouped_topk(hidden_states: torch.Tensor,
441 return topk_weights, topk_ids
442
443
440 -def fused_marlin_moe(hidden_states: torch.Tensor,
441 - w1: torch.Tensor,
442 - w2: torch.Tensor,
443 - gating_output: torch.Tensor,
444 - g_idx1: torch.Tensor,
445 - g_idx2: torch.Tensor,
446 - rand_perm1: torch.Tensor,
447 - rand_perm2: torch.Tensor,
448 - topk: int,
449 - renormalize: bool,
450 - override_config: Optional[Dict[str, Any]] = None,
451 - use_fp8: bool = False,
452 - w1_scale: Optional[torch.Tensor] = None,
453 - w2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
454 - """
455 - This function computes a Mixture of Experts (MoE) layer using two sets of
456 - weights, w1 and w2, and top-k gating mechanism.
457 - Parameters:
458 - - hidden_states (torch.Tensor): The input tensor to the MoE layer.
459 - - w1 (torch.Tensor): The first set of expert weights.
460 - - w2 (torch.Tensor): The second set of expert weights.
461 - - gating_output (torch.Tensor): The output of the gating operation
462 - (before softmax).
463 - - topk (int): The number of top-k experts to select.
464 - - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
465 - - inplace (bool): If True, perform the operation in-place.
466 - Defaults to False.
467 - - override_config (Optional[Dict[str, Any]]): Optional override
468 - for the kernel configuration.
469 - - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
470 - products for w1 and w2. Defaults to False.
471 - - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
472 - w1.
473 - - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
474 - w2.
475 - Returns:
476 - - torch.Tensor: The output tensor after applying the MoE layer.
477 - """
478 - # Check constraints.
479 - assert hidden_states.shape[0] == gating_output.shape[0], (
480 - "Number of tokens mismatch")
481 - assert hidden_states.shape[
482 - 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
483 - assert hidden_states.shape[
484 - 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
485 - assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
486 - assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
487 - assert w1.is_contiguous(), "Expert weights1 must be contiguous"
488 - assert w2.is_contiguous(), "Expert weights2 must be contiguous"
489 - assert hidden_states.dtype in [
490 - torch.float32, torch.float16, torch.bfloat16
491 - ]
492 -
493 - #TODO fp8 is not implemented yet
494 - assert not use_fp8
495 -
496 - M, K = hidden_states.shape
497 - E = w1.shape[0]
498 - N = w2.shape[1] * 16
499 -
500 - topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
501 - renormalize)
502 -
503 - get_config_func = functools.partial(try_get_optimal_moe_config,
504 - w1.shape,
505 - w2.shape,
506 - topk_ids.shape[1],
507 - "float8" if use_fp8 else None,
508 - override_config=override_config,
509 - is_marlin=True)
510 - config = get_config_func(M)
511 -
512 - block_size_m = config['BLOCK_SIZE_M']
513 -
514 - sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
515 -
516 - max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
517 - workspace = torch.zeros(max_workspace_size,
518 - dtype=torch.int,
519 - device="cuda",
520 - requires_grad=False)
521 -
522 - intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N),
523 - device=hidden_states.device,
524 - dtype=hidden_states.dtype)
525 -
526 - intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe(
527 - hidden_states, w1, sorted_token_ids, topk_weights, topk_ids, w1_scale,
528 - g_idx1, rand_perm1, workspace, M, 2 * N, K, True, E, topk,
529 - block_size_m, True, False)
530 -
531 - ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
532 -
533 - intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
534 - intermediate_cache2, w2, sorted_token_ids, topk_weights, topk_ids,
535 - w2_scale, g_idx2, rand_perm2, workspace, M, K, N, True, E, topk,
536 - block_size_m, False, True)
537 -
538 - return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
539 - dim=1)
540 -
541 -
444 def get_config_dtype_str(dtype: torch.dtype,
445 use_int8_w8a16: Optional[bool] = False,
446 use_fp8_w8a8: Optional[bool] = False):
vllm/model_executor/layers/fused_moe/layer.py
+36 -170
@@ -1,5 +1,4 @@
1 from abc import abstractmethod
2 -from enum import Enum
2 from typing import List, Optional, Tuple
3
4 import torch
@@ -16,12 +15,6 @@ from vllm.model_executor.utils import set_weight_attrs
15 logger = init_logger(__name__)
16
17
19 -class FusedMoeWeightScaleSupported(Enum):
20 - TENSOR = "tensor"
21 - CHANNEL = "channel"
22 - GROUP = "group"
23 -
24 -
18 class FusedMoEMethodBase(QuantizeMethodBase):
19
20 @abstractmethod
@@ -206,182 +199,55 @@ class FusedMoE(torch.nn.Module):
199 params_dtype=params_dtype,
200 weight_loader=self.weight_loader)
201
209 - def _load_per_tensor_weight_scale(self, shard_id: str,
210 - param: torch.nn.Parameter,
211 - loaded_weight: torch.Tensor,
212 - expert_id: int):
213 - param_data = param.data
214 - # for per tensor weight quantization
215 - if shard_id in ("w1", "w3"):
216 - # We have to keep the weight scales of w1 and w3 because
217 - # we need to re-quantize w1/w3 weights after weight loading.
218 - idx = 0 if shard_id == "w1" else 1
219 - param_data[expert_id][idx] = loaded_weight
220 - # If we are in the row parallel case (down_proj)
221 - elif shard_id == "w2":
222 - param_data[expert_id] = loaded_weight
223 -
224 - def _load_model_weight_or_group_weight_scale(self, shard_dim: int,
225 - expert_data: torch.Tensor,
226 - shard_id: str,
227 - loaded_weight: torch.tensor,
228 - tp_rank: int):
229 - # Load grouped weight scales for group quantization
230 - # or model weights
231 - if shard_id == "w2":
232 - self._load_w2(shard_id=shard_id,
233 - shard_dim=shard_dim,
234 - loaded_weight=loaded_weight,
235 - expert_data=expert_data,
236 - tp_rank=tp_rank)
237 - elif shard_id in ("w1", "w3"):
238 - self._load_w13(shard_id=shard_id,
239 - shard_dim=shard_dim,
240 - loaded_weight=loaded_weight,
241 - expert_data=expert_data,
242 - tp_rank=tp_rank)
243 -
244 - def _load_per_channel_weight_scale(self, expert_data: torch.Tensor,
245 - shard_dim: int, shard_id: str,
246 - loaded_weight: torch.tensor,
247 - tp_rank: int):
248 - # for per channel weight quantization
249 - if shard_id == "w2":
250 - expert_data.copy_(loaded_weight)
251 - elif shard_id in ("w1", "w3"):
252 - self._load_w13(shard_id=shard_id,
253 - shard_dim=shard_dim,
254 - loaded_weight=loaded_weight,
255 - expert_data=expert_data,
256 - tp_rank=tp_rank)
257 -
258 - def _load_w13(self, expert_data: torch.Tensor, shard_dim: int,
259 - shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
260 -
261 - # Index the loaded weight for tp sharding.
262 - # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
263 - shard_size = expert_data.shape[shard_dim] // 2
264 - loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
265 - shard_size)
266 - # Narrow parameter and load.
267 - # w1, gate_proj: Load into first logical weight of w13.
268 - if shard_id == "w1":
269 - expert_data = expert_data.narrow(shard_dim, 0, shard_size)
270 - # w3, up_proj: Load into second logical weight of w13.
271 - else:
272 - assert shard_id == "w3"
273 - expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
274 - expert_data.copy_(loaded_weight)
275 -
276 - def _load_w2(self, expert_data: torch.Tensor, shard_dim: int,
277 - shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
278 -
279 - # Index the loaded weight for tp sharding.
280 - # down_proj: "RowParallel" so tp sharding on input_dim
281 - # Narrow parameter and load.
282 - shard_size = expert_data.shape[shard_dim]
283 - loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
284 - shard_size)
285 - # w2, down_proj: Load into only logical weight of w2.
286 - expert_data.copy_(loaded_weight)
287 -
288 - def _load_single_value(self, param: torch.nn.Parameter,
289 - loaded_weight: torch.Tensor, expert_id: int):
290 - param_data = param.data
291 -
292 - # Input scales can be loaded directly and should be equal.
293 - param_data[expert_id] = loaded_weight
294 -
202 def weight_loader(self, param: torch.nn.Parameter,
203 loaded_weight: torch.Tensor, weight_name: str,
204 shard_id: str, expert_id: int) -> None:
298 -
205 if shard_id not in ("w1", "w2", "w3"):
206 raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
207 f"got {shard_id}.")
208
303 - WEIGHT_SCALE_SUPPORTED = [
304 - e.value for e in FusedMoeWeightScaleSupported
305 - ]
306 - # Fetch the dim to shard the parameter/loaded weight
307 - # based on the shard id. This will be whatever
308 - # dimension intermediate_size is used.
309 - SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
209 + # Special case for fp8 scales.
210 + if getattr(param, "is_fp8_scale", False):
211 + self._load_fp8_scale(param.data, loaded_weight, weight_name,
212 + shard_id, expert_id)
213 + return
214
215 expert_data = param.data[expert_id]
216 tp_rank = get_tensor_model_parallel_rank()
217
314 - # is_transposed: whether or not the parameter is transposed on disk
315 - # If transposed, the loaded weight will be transposed and the dim
316 - # to shard the loaded weight will be flipped.
317 - is_transposed = getattr(param, "is_transposed", False)
318 - shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
319 - if is_transposed:
320 - loaded_weight = loaded_weight.t().contiguous()
321 - shard_dim = ~shard_dim
322 -
323 - # Case weight_scales
324 - if "weight_scale" in weight_name:
325 - # load the weight scaling based on the quantization scheme
326 - # supported weight scales can be found in
327 - # FusedMoeWeightScaleSupported
328 - # TODO @dsikka: once hardened, refactor to use vLLM Parameters
329 - # specific to each case
330 - quant_method = getattr(param, "quant_method", None)
331 - if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value:
332 - self._load_per_channel_weight_scale(
333 - shard_id=shard_id,
334 - shard_dim=shard_dim,
335 - loaded_weight=loaded_weight,
336 - expert_data=expert_data,
337 - tp_rank=tp_rank)
338 - elif quant_method == FusedMoeWeightScaleSupported.GROUP.value:
339 - self._load_model_weight_or_group_weight_scale(
340 - shard_id=shard_id,
341 - shard_dim=shard_dim,
342 - loaded_weight=loaded_weight,
343 - expert_data=expert_data,
344 - tp_rank=tp_rank)
345 - elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
346 - self._load_per_tensor_weight_scale(shard_id=shard_id,
347 - param=param,
348 - loaded_weight=loaded_weight,
349 - expert_id=expert_id)
350 - else:
351 - raise ValueError(
352 - f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}")
353 - return
354 -
355 - if "weight_shape" in weight_name:
356 - self._load_single_value(param=param,
357 - loaded_weight=loaded_weight,
358 - expert_id=expert_id)
359 - return
218 + # If transposed, weight is saved as [input_dim, output_dim]
219 + # Otherwise, weight is saved as [output_dim, input_dim]
220 + # Default is not transposed/input dim is dim 1
221 + input_dim = getattr(param, "input_dim", 1)
222 + output_dim = getattr(param, "output_dim", 0)
223
361 - # Case input scale
362 - if "input_scale" in weight_name:
363 - # Note: input_scale loading is only supported for fp8
364 - if param.data[expert_id] != 1 and (param.data[expert_id] -
365 - loaded_weight).abs() > 1e-5:
366 - raise ValueError(
367 - "input_scales of w1 and w3 of a layer "
368 - f"must be equal. But got {param.data[expert_id]} "
369 - f"vs. {loaded_weight}")
370 -
371 - self._load_single_value(param=param,
372 - loaded_weight=loaded_weight,
373 - expert_id=expert_id)
374 - return
224 + # Index the loaded weight for tp sharding.
225 + # down_proj: "RowParallel" so tp sharding on input_dim
226 + if shard_id == "w2":
227 + shard_dim = input_dim
228 + shard_size = expert_data.shape[shard_dim]
229 + # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
230 + elif shard_id in ("w1", "w3"):
231 + shard_dim = output_dim
232 + shard_size = expert_data.shape[output_dim] // 2
233 + offset = shard_size * tp_rank
234 + loaded_weight = loaded_weight.narrow(shard_dim, offset, shard_size)
235
376 - # Case model weights
377 - if "weight" in weight_name:
378 - self._load_model_weight_or_group_weight_scale(
379 - shard_id=shard_id,
380 - shard_dim=shard_dim,
381 - loaded_weight=loaded_weight,
382 - expert_data=expert_data,
383 - tp_rank=tp_rank)
384 - return
236 + # Narrow parameter and load.
237 + # w1, gate_proj: Load into first logical weight of w13.
238 + if shard_id == "w1":
239 + expert_data = expert_data.narrow(shard_dim, 0, shard_size)
240 + expert_data.copy_(loaded_weight)
241 + # w3, up_proj: Load into second logical weight of w13.
242 + elif shard_id == "w3":
243 + expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
244 + expert_data.copy_(loaded_weight)
245 + # w2, down_proj: Load into only logical weight of w2.
246 + elif shard_id == "w2":
247 + expert_data.copy_(loaded_weight)
248 + else:
249 + raise ValueError(
250 + f"Expected shard_id w1,w2 or w3 but got {shard_id}")
251
252 @staticmethod
253 def select_experts(hidden_states: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
-5
@@ -3,12 +3,9 @@ from typing import Any, Dict, List, Optional
3 import torch
4 from pydantic import BaseModel
5
6 -from vllm.model_executor.layers.fused_moe import FusedMoE
6 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7 from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8 QuantizationConfig, QuantizeMethodBase)
10 -from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
11 - CompressedTensorsMoEMethod)
9 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11 CompressedTensorsScheme, CompressedTensorsUnquantized,
@@ -67,8 +64,6 @@ class CompressedTensorsConfig(QuantizationConfig):
64 return CompressedTensorsLinearMethod(self)
65 if isinstance(layer, Attention):
66 return CompressedTensorsKVCacheMethod(self)
70 - if isinstance(layer, FusedMoE):
71 - return CompressedTensorsMoEMethod(self)
67 return None
68
69 @classmethod
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py deleted
-283
@@ -1,283 +0,0 @@
1 -import enum
2 -from enum import Enum
3 -from typing import List, Optional
4 -
5 -import torch
6 -
7 -from vllm import _custom_ops as ops
8 -from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
9 -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10 - WNA16_SUPPORTED_BITS)
11 -from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
12 - CompressionFormat)
13 -from vllm.model_executor.utils import set_weight_attrs
14 -
15 -
16 -class GPTQMarlinState(Enum):
17 - REPACK = enum.auto()
18 - READY = enum.auto()
19 -
20 -
21 -__all__ = ["CompressedTensorsMoEMethod"]
22 -
23 -
24 -class CompressedTensorsMoEMethod(FusedMoEMethodBase):
25 -
26 - def __init__(
27 - self,
28 - quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501
29 - ):
30 - self.quant_config = quant_config
31 - # TODO: @dsikka: refactor this to use schemes as other kernels
32 - # are supported + check if the layer is being ignored.
33 - config = self.quant_config.target_scheme_map["Linear"].get("weights")
34 - self.num_bits = config.num_bits
35 - self.packed_factor = 32 // config.num_bits
36 - self.strategy = config.strategy.value
37 - self.group_size = config.group_size
38 - assert config.symmetric, (
39 - "Only symmetric quantization is supported for MoE")
40 -
41 - if not (self.quant_config.quant_format
42 - == CompressionFormat.pack_quantized.value
43 - and self.num_bits in WNA16_SUPPORTED_BITS):
44 - raise ValueError("For Fused MoE layers, only ",
45 - f"{CompressionFormat.pack_quantized.value} ",
46 - "is supported for the following bits: ",
47 - f"{WNA16_SUPPORTED_BITS}")
48 -
49 - def create_weights(self, layer: torch.nn.Module, num_experts: int,
50 - hidden_size: int, intermediate_size: int,
51 - params_dtype: torch.dtype, **extra_weight_attrs):
52 -
53 - # Will transpose the loaded weight along the
54 - # intermediate and hidden dim sizes. Will
55 - # shard for TP along the transposed dims
56 - extra_weight_attrs.update({
57 - "is_transposed": True,
58 - "quant_method": self.strategy
59 - })
60 - w13_weight = torch.nn.Parameter(torch.empty(num_experts,
61 - hidden_size //
62 - self.packed_factor,
63 - 2 * intermediate_size,
64 - dtype=torch.int32),
65 - requires_grad=False)
66 - layer.register_parameter("w13_weight_packed", w13_weight)
67 - set_weight_attrs(w13_weight, extra_weight_attrs)
68 -
69 - w2_weight = torch.nn.Parameter(torch.empty(num_experts,
70 - intermediate_size //
71 - self.packed_factor,
72 - hidden_size,
73 - dtype=torch.int32),
74 - requires_grad=False)
75 - layer.register_parameter("w2_weight_packed", w2_weight)
76 - set_weight_attrs(w2_weight, extra_weight_attrs)
77 -
78 - if self.strategy == "channel":
79 - num_groups_w2 = num_groups_w13 = 1
80 - self.group_size = -1
81 - else:
82 - num_groups_w2 = intermediate_size // self.group_size
83 - num_groups_w13 = hidden_size // self.group_size
84 -
85 - w13_scale = torch.nn.Parameter(torch.ones(num_experts,
86 - num_groups_w13,
87 - 2 * intermediate_size,
88 - dtype=params_dtype),
89 - requires_grad=False)
90 - layer.register_parameter("w13_weight_scale", w13_scale)
91 - set_weight_attrs(w13_scale, extra_weight_attrs)
92 -
93 - w2_scale = torch.nn.Parameter(torch.ones(num_experts,
94 - num_groups_w2,
95 - hidden_size,
96 - dtype=params_dtype),
97 - requires_grad=False)
98 - layer.register_parameter("w2_weight_scale", w2_scale)
99 - set_weight_attrs(w2_scale, extra_weight_attrs)
100 -
101 - w2_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
102 - requires_grad=False)
103 - layer.register_parameter("w2_weight_shape", w2_weight_shape)
104 - set_weight_attrs(w2_weight_shape, extra_weight_attrs)
105 - w13_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
106 - requires_grad=False)
107 -
108 - layer.register_parameter("w13_weight_shape", w13_weight_shape)
109 - set_weight_attrs(w13_weight_shape, extra_weight_attrs)
110 -
111 - w13_g_idx = torch.nn.Parameter(
112 - torch.empty(
113 - num_experts,
114 - hidden_size,
115 - dtype=torch.int32,
116 - ),
117 - requires_grad=False,
118 - )
119 - layer.register_parameter("w13_g_idx", w13_g_idx)
120 - set_weight_attrs(w13_g_idx, extra_weight_attrs)
121 -
122 - w2_g_idx = torch.nn.Parameter(
123 - torch.empty(
124 - num_experts,
125 - intermediate_size,
126 - dtype=torch.int32,
127 - ),
128 - requires_grad=False,
129 - )
130 - layer.register_parameter("w2_g_idx", w2_g_idx)
131 - set_weight_attrs(w2_g_idx, extra_weight_attrs)
132 -
133 - w13_g_idx_sort_indices = torch.nn.Parameter(
134 - torch.empty(
135 - num_experts,
136 - hidden_size,
137 - dtype=torch.int32,
138 - ),
139 - requires_grad=False,
140 - )
141 - layer.register_parameter("w13_g_idx_sort_indices",
142 - w13_g_idx_sort_indices)
143 - set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
144 -
145 - w2_g_idx_sort_indices = torch.nn.Parameter(
146 - torch.empty(
147 - num_experts,
148 - intermediate_size,
149 - dtype=torch.int32,
150 - ),
151 - requires_grad=False,
152 - )
153 - layer.register_parameter("w2_g_idx_sort_indices",
154 - w2_g_idx_sort_indices)
155 - set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
156 -
157 - layer.a13_scale = None
158 - layer.a2_scale = None
159 - layer.marlin_state = GPTQMarlinState.REPACK
160 -
161 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
162 -
163 - def replace_tensor(name, new_t):
164 - # It is important to use resize_() here since it ensures
165 - # the same buffer is reused
166 - getattr(layer, name).resize_(new_t.shape)
167 - getattr(layer, name).copy_(new_t)
168 - del new_t
169 -
170 - def get_scale_perms(num_bits: int):
171 - scale_perm: List[int] = []
172 - for i in range(8):
173 - scale_perm.extend([i + 8 * j for j in range(8)])
174 - scale_perm_single: List[int] = []
175 - for i in range(4):
176 - scale_perm_single.extend(
177 - [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
178 - return scale_perm, scale_perm_single
179 -
180 - def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
181 - group_size: int, num_bits: int):
182 - scale_perm, scale_perm_single = get_scale_perms(num_bits)
183 - if group_size < size_k and group_size != -1:
184 - s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
185 - else:
186 - s = s.reshape((-1, len(scale_perm_single)))[:,
187 - scale_perm_single]
188 - s = s.reshape((-1, size_n)).contiguous()
189 - return s
190 -
191 - def marlin_moe_permute_scales(s: torch.Tensor, size_k: int,
192 - size_n: int, group_size: int,
193 - num_bits: int):
194 - num_experts = s.shape[0]
195 - output = torch.empty((num_experts, s.shape[1], s.shape[2]),
196 - device=s.device,
197 - dtype=s.dtype)
198 - for e in range(num_experts):
199 - output[e] = marlin_permute_scales(s[e], size_k, size_n,
200 - group_size, num_bits)
201 - return output
202 -
203 - size_k2 = layer.w2_weight_packed.shape[2]
204 - size_k13 = layer.w13_weight_packed.shape[2]
205 -
206 - num_experts = layer.w13_g_idx.shape[0]
207 - device = layer.w13_g_idx.device
208 - layer.w13_g_idx = torch.nn.Parameter(
209 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
210 - requires_grad=False,
211 - )
212 - layer.w2_g_idx = torch.nn.Parameter(
213 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
214 - requires_grad=False,
215 - )
216 - layer.w13_g_idx_sort_indices = torch.nn.Parameter(
217 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
218 - requires_grad=False,
219 - )
220 - layer.w2_g_idx_sort_indices = torch.nn.Parameter(
221 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
222 - requires_grad=False,
223 - )
224 -
225 - marlin_w13_qweight = ops.gptq_marlin_moe_repack(
226 - layer.w13_weight_packed,
227 - layer.w13_g_idx_sort_indices,
228 - layer.w13_weight_packed.shape[1] * self.packed_factor,
229 - layer.w13_weight_packed.shape[2],
230 - self.num_bits,
231 - )
232 - replace_tensor("w13_weight_packed", marlin_w13_qweight)
233 - marlin_w2_qweight = ops.gptq_marlin_moe_repack(
234 - layer.w2_weight_packed,
235 - layer.w2_g_idx_sort_indices,
236 - layer.w2_weight_packed.shape[1] * self.packed_factor,
237 - layer.w2_weight_packed.shape[2],
238 - self.num_bits,
239 - )
240 - replace_tensor("w2_weight_packed", marlin_w2_qweight)
241 - # Repack scales
242 - marlin_w13_scales = marlin_moe_permute_scales(
243 - layer.w13_weight_scale,
244 - size_k13,
245 - layer.w13_weight_scale.shape[2],
246 - self.group_size,
247 - self.num_bits,
248 - )
249 - replace_tensor("w13_weight_scale", marlin_w13_scales)
250 - marlin_w2_scales = marlin_moe_permute_scales(
251 - layer.w2_weight_scale,
252 - layer.w2_weight_scale.shape[1] * self.packed_factor,
253 - size_k2,
254 - self.group_size,
255 - self.num_bits,
256 - )
257 - replace_tensor("w2_weight_scale", marlin_w2_scales)
258 -
259 - def apply(self,
260 - layer: torch.nn.Module,
261 - x: torch.Tensor,
262 - router_logits: torch.Tensor,
263 - top_k: int,
264 - renormalize: bool = True,
265 - use_grouped_topk: bool = False,
266 - num_expert_group: Optional[int] = None,
267 - topk_group: Optional[int] = None) -> torch.Tensor:
268 -
269 - from vllm.model_executor.layers.fused_moe.fused_moe import (
270 - fused_marlin_moe)
271 -
272 - return fused_marlin_moe(x,
273 - layer.w13_weight_packed,
274 - layer.w2_weight_packed,
275 - router_logits,
276 - layer.w13_g_idx,
277 - layer.w2_g_idx,
278 - layer.w13_g_idx_sort_indices,
279 - layer.w2_g_idx_sort_indices,
280 - top_k,
281 - renormalize=renormalize,
282 - w1_scale=layer.w13_weight_scale,
283 - w2_scale=layer.w2_weight_scale)
vllm/model_executor/layers/quantization/fp8.py
+18 -11
@@ -7,8 +7,7 @@ from torch.nn.parameter import Parameter
7 import vllm.envs as envs
8 from vllm import _custom_ops as ops
9 from vllm.logger import init_logger
10 -from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11 - FusedMoeWeightScaleSupported)
10 +from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
11 from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12 UnquantizedLinearMethod)
13 from vllm.model_executor.layers.quantization.base_config import (
@@ -319,16 +318,19 @@ class Fp8MoEMethod(FusedMoEMethodBase):
318 dtype=torch.float32),
319 requires_grad=False)
320 layer.register_parameter("w2_weight_scale", w2_weight_scale)
322 - # Add the quantization method used (per tensor/grouped/channel)
323 - # to ensure the weight scales are loaded in properly
324 - extra_weight_attrs.update(
325 - {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
321 +
322 # If loading fp8 checkpoint, pass the weight loaders.
323 # If loading an fp16 checkpoint, do not (we will quantize in
324 # process_weights_after_loading()
325 if self.quant_config.is_checkpoint_fp8_serialized:
330 - set_weight_attrs(w13_weight_scale, extra_weight_attrs)
331 - set_weight_attrs(w2_weight_scale, extra_weight_attrs)
326 + set_weight_attrs(w13_weight_scale, {
327 + "is_fp8_scale": True,
328 + **extra_weight_attrs
329 + })
330 + set_weight_attrs(w2_weight_scale, {
331 + "is_fp8_scale": True,
332 + **extra_weight_attrs
333 + })
334
335 # INPUT_SCALES
336 if self.quant_config.activation_scheme == "static":
@@ -341,14 +343,19 @@ class Fp8MoEMethod(FusedMoEMethodBase):
343 num_experts, dtype=torch.float32),
344 requires_grad=False)
345 layer.register_parameter("w13_input_scale", w13_input_scale)
344 - set_weight_attrs(w13_input_scale, extra_weight_attrs)
346 + set_weight_attrs(w13_input_scale, {
347 + "is_fp8_scale": True,
348 + **extra_weight_attrs
349 + })
350
351 w2_input_scale = torch.nn.Parameter(torch.ones(
352 num_experts, dtype=torch.float32),
353 requires_grad=False)
354 layer.register_parameter("w2_input_scale", w2_input_scale)
350 - set_weight_attrs(w2_input_scale, extra_weight_attrs)
351 -
355 + set_weight_attrs(w2_input_scale, {
356 + "is_fp8_scale": True,
357 + **extra_weight_attrs
358 + })
359 else:
360 layer.w13_input_scale = None
361 layer.w2_input_scale = None
vllm/model_executor/model_loader/utils.py
+2 -2
@@ -23,11 +23,11 @@ def get_model_architecture(
23 architectures = getattr(model_config.hf_config, "architectures", [])
24 # Special handling for quantized Mixtral.
25 # FIXME(woosuk): This is a temporary hack.
26 - mixtral_supported = ["fp8", "compressed-tensors"]
26 if (model_config.quantization is not None
28 - and model_config.quantization not in mixtral_supported
27 + and model_config.quantization != "fp8"
28 and "MixtralForCausalLM" in architectures):
29 architectures = ["QuantMixtralForCausalLM"]
30 +
31 return ModelRegistry.resolve_model_cls(architectures)
32
33
vllm/model_executor/models/jamba.py
+1 -1
@@ -920,7 +920,7 @@ class JambaForCausalLM(nn.Module, HasInnerState):
920 weight_loader = param.weight_loader
921 weight_loader(param,
922 loaded_weight,
923 - name,
923 + weight_name,
924 shard_id=shard_id,
925 expert_id=expert_id)
926 break
vllm/model_executor/models/mixtral.py
-1
@@ -73,7 +73,6 @@ class MixtralMoE(nn.Module):
73 self.hidden_size = hidden_size
74
75 # Gate always runs at half / full precision for now.
76 -
76 self.gate = ReplicatedLinear(hidden_size,
77 num_experts,
78 bias=False,