A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
Revert "[Kernel] Expand MoE weight loading + Add Fused Marlin MoE Kernel (#7527)" (#7764)
Michael Goin committed
Aug 21, 2024 at 23:42 UTC
aae74ef95c370df92584e08939a15707ea8a5d3f
15 files changed
+84
-2374
CMakeLists.txt
+1
-2
@@ -286,8 +286,7 @@ define_gpu_extension_target(
286
287
set(VLLM_MOE_EXT_SRC
288
"csrc/moe/torch_bindings.cpp"
289
- "csrc/moe/topk_softmax_kernels.cu"
290
- "csrc/moe/marlin_moe_ops.cu")
289
+ "csrc/moe/topk_softmax_kernels.cu")
290
291
define_gpu_extension_target(
292
_moe_C
csrc/moe/marlin_moe_ops.cu
deleted
-1740
@@ -1,1740 +0,0 @@
1
-/*
2
- * Modified by Neural Magic
3
- * Copyright (C) Marlin.2024 Elias Frantar
4
- *
5
- * Licensed under the Apache License, Version 2.0 (the "License");
6
- * you may not use this file except in compliance with the License.
7
- * You may obtain a copy of the License at
8
- *
9
- * http://www.apache.org/licenses/LICENSE-2.0
10
- *
11
- * Unless required by applicable law or agreed to in writing, software
12
- * distributed under the License is distributed on an "AS IS" BASIS,
13
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- * See the License for the specific language governing permissions and
15
- * limitations under the License.
16
- */
17
-
18
-#include <torch/all.h>
19
-
20
-#include <ATen/cuda/CUDAContext.h>
21
-#include <c10/cuda/CUDAGuard.h>
22
-#include <cuda.h>
23
-#include <cuda_fp16.h>
24
-#include <cuda_runtime.h>
25
-
26
-#include <iostream>
27
-
28
-template <typename T>
29
-inline std::string str(T x) {
30
- return std::to_string(x);
31
-}
32
-
33
-namespace marlin_moe {
34
-
35
-constexpr int ceildiv(int a, int b) { return (a + b - 1) / b; }
36
-
37
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
38
-
39
-// Instances of `Vec` are used to organize groups of >>registers<<, as needed
40
-// for instance as inputs to tensor core operations. Consequently, all
41
-// corresponding index accesses must be compile-time constants, which is why we
42
-// extensively use `#pragma unroll` throughout the kernel code to guarantee
43
-// this.
44
-template <typename T, int n>
45
-struct Vec {
46
- T elems[n];
47
- __device__ T& operator[](int i) { return elems[i]; }
48
-};
49
-
50
-using I4 = Vec<int, 4>;
51
-
52
-// Matrix fragments for tensor core instructions; their precise layout is
53
-// documented here:
54
-// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type
55
-using FragA = Vec<half2, 4>;
56
-using FragB = Vec<half2, 2>;
57
-using FragC = Vec<float, 4>;
58
-using FragS = Vec<half2, 1>; // quantization scales
59
-
60
-// Predicated asynchronous global->shared copy; used for inputs A where we apply
61
-// predication to handle batchsizes that are not multiples of 16.
62
-__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr,
63
- bool pred = true) {
64
- const int BYTES = 16;
65
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
66
- asm volatile(
67
- "{\n"
68
- " .reg .pred p;\n"
69
- " setp.ne.b32 p, %0, 0;\n"
70
- " @p cp.async.cg.shared.global [%1], [%2], %3;\n"
71
- "}\n" ::"r"((int)pred),
72
- "r"(smem), "l"(glob_ptr), "n"(BYTES));
73
-}
74
-
75
-// Asynchronous global->shared copy
76
-__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) {
77
- const int BYTES = 16;
78
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
79
- asm volatile(
80
- "{\n"
81
- " cp.async.cg.shared.global [%0], [%1], %2;\n"
82
- "}\n" ::"r"(smem),
83
- "l"(glob_ptr), "n"(BYTES));
84
-}
85
-
86
-// Async copy fence.
87
-__device__ inline void cp_async_fence() {
88
- asm volatile("cp.async.commit_group;\n" ::);
89
-}
90
-
91
-// Wait until at most `n` async copy stages are still pending.
92
-template <int n>
93
-__device__ inline void cp_async_wait() {
94
- asm volatile("cp.async.wait_group %0;\n" ::"n"(n));
95
-}
96
-
97
-// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
98
-// output/accumulation.
99
-__device__ inline void mma(const FragA& a_frag, const FragB& frag_b,
100
- FragC& frag_c) {
101
- const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
102
- const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
103
- float* c = reinterpret_cast<float*>(&frag_c);
104
- asm volatile(
105
- "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
106
- "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
107
- : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
108
- : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
109
- "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
110
-}
111
-
112
-// Instruction for loading a full 16x16 matrix fragment of operand A from shared
113
-// memory, directly in tensor core layout.
114
-__device__ inline void ldsm4(FragA& frag_a, const void* smem_ptr) {
115
- uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
116
- uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
117
- asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
118
- : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
119
- : "r"(smem));
120
-}
121
-
122
-// Lookup-table based 3-input logical operation; explicitly used for
123
-// dequantization as the compiler does not seem to automatically recognize it in
124
-// all cases.
125
-template <int lut>
126
-__device__ inline int lop3(int a, int b, int c) {
127
- int res;
128
- asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n"
129
- : "=r"(res)
130
- : "r"(a), "r"(b), "r"(c), "n"(lut));
131
- return res;
132
-}
133
-
134
-// Efficiently dequantize an int32 value into a full B-fragment of 4 fp16
135
-// values. We mostly follow the strategy in the link below, with some small
136
-// changes:
137
-// https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h
138
-__device__ inline FragB dequant(int q) {
139
- const int LO = 0x000f000f;
140
- const int HI = 0x00f000f0;
141
- const int EX = 0x64006400;
142
- // Guarantee that the `(a & b) | c` operations are LOP3s.
143
- int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX);
144
- int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX);
145
- // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point
146
- // directly into `SUB` and `ADD`.
147
- const int SUB = 0x64086408;
148
- const int MUL = 0x2c002c00;
149
- const int ADD = 0xd480d480;
150
- FragB frag_b;
151
- frag_b[0] = __hsub2(*reinterpret_cast<half2*>(&lo),
152
- *reinterpret_cast<const half2*>(&SUB));
153
- frag_b[1] = __hfma2(*reinterpret_cast<half2*>(&hi),
154
- *reinterpret_cast<const half2*>(&MUL),
155
- *reinterpret_cast<const half2*>(&ADD));
156
- return frag_b;
157
-}
158
-
159
-// Multiply dequantized values by the corresponding quantization scale; used
160
-// only for grouped quantization.
161
-__device__ inline void scale(FragB& frag_b, FragS& frag_s, int i) {
162
- half2 s = __half2half2(reinterpret_cast<__half*>(&frag_s)[i]);
163
- frag_b[0] = __hmul2(frag_b[0], s);
164
- frag_b[1] = __hmul2(frag_b[1], s);
165
-}
166
-
167
-// Given 2 floats multiply by 2 scales (halves)
168
-__device__ inline void scale_float(float* c, FragS& s) {
169
- __half* s_ptr = reinterpret_cast<__half*>(&s);
170
- c[0] = __fmul_rn(c[0], __half2float(s_ptr[0]));
171
- c[1] = __fmul_rn(c[1], __half2float(s_ptr[1]));
172
-}
173
-
174
-// Same as above, but for act_order (each K is multiplied individually)
175
-__device__ inline void scale4(FragB& frag_b, FragS& frag_s_1, FragS& frag_s_2,
176
- FragS& frag_s_3, FragS& frag_s_4, int i) {
177
- __half2 s_val_1_2;
178
- s_val_1_2.x = reinterpret_cast<__half*>(&frag_s_1)[i];
179
- s_val_1_2.y = reinterpret_cast<__half*>(&frag_s_2)[i];
180
-
181
- __half2 s_val_3_4;
182
- s_val_3_4.x = reinterpret_cast<__half*>(&frag_s_3)[i];
183
- s_val_3_4.y = reinterpret_cast<__half*>(&frag_s_4)[i];
184
-
185
- frag_b[0] = __hmul2(frag_b[0], s_val_1_2);
186
- frag_b[1] = __hmul2(frag_b[1], s_val_3_4);
187
-}
188
-
189
-// Wait until barrier reaches `count`, then lock for current threadblock.
190
-__device__ inline void barrier_acquire(int* lock, int count) {
191
- if (threadIdx.x == 0) {
192
- int state = -1;
193
- do
194
- // Guarantee that subsequent writes by this threadblock will be visible
195
- // globally.
196
- asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
197
- : "=r"(state)
198
- : "l"(lock));
199
- while (state != count);
200
- }
201
- __syncthreads();
202
-}
203
-
204
-// Release barrier and increment visitation count.
205
-__device__ inline void barrier_release(int* lock, bool reset = false) {
206
- __syncthreads();
207
- if (threadIdx.x == 0) {
208
- if (reset) {
209
- lock[0] = 0;
210
- return;
211
- }
212
- int val = 1;
213
- // Make sure that all writes since acquiring this barrier are visible
214
- // globally, while releasing the barrier.
215
- asm volatile("fence.acq_rel.gpu;\n");
216
- asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
217
- :
218
- : "l"(lock), "r"(val));
219
- }
220
-}
221
-
222
-// For a given "a" of size [M,K] performs a permutation of the K columns based
223
-// on the given "perm" indices.
224
-__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
225
- int const* __restrict__ perm_int_ptr,
226
- int4* __restrict__ out_int4_ptr, int size_m,
227
- int size_k, int block_rows) {
228
- int start_row = block_rows * blockIdx.x;
229
- int finish_row = start_row + block_rows;
230
- if (finish_row > size_m) {
231
- finish_row = size_m;
232
- }
233
- int cur_block_rows = finish_row - start_row;
234
-
235
- int row_stride = size_k * sizeof(half) / 16;
236
-
237
- auto permute_row = [&](int row) {
238
- int iters = size_k / blockDim.x;
239
- int rest = size_k % blockDim.x;
240
-
241
- int offset = row * row_stride;
242
-
243
- half const* a_row_half = reinterpret_cast<half const*>(a_int4_ptr + offset);
244
- half* out_half = reinterpret_cast<half*>(out_int4_ptr + offset);
245
-
246
- int base_k = 0;
247
-
248
- for (int i = 0; i < iters; i++) {
249
- int cur_k = base_k + threadIdx.x;
250
- int src_pos = perm_int_ptr[cur_k];
251
-
252
- out_half[cur_k] = a_row_half[src_pos];
253
-
254
- base_k += blockDim.x;
255
- }
256
-
257
- if (rest) {
258
- if (threadIdx.x < rest) {
259
- int cur_k = base_k + threadIdx.x;
260
- int src_pos = perm_int_ptr[cur_k];
261
-
262
- out_half[cur_k] = a_row_half[src_pos];
263
- }
264
- }
265
- };
266
-
267
- for (int i = 0; i < cur_block_rows; i++) {
268
- int cur_row = start_row + i;
269
- if (cur_row < size_m) {
270
- permute_row(cur_row);
271
- }
272
- }
273
-}
274
-
275
-__global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
276
- int* __restrict__ expert_offsets,
277
- int topk_length, int block_size) {
278
- int expert_id = threadIdx.x;
279
- int num_experts = blockDim.x;
280
-
281
- int occurrences = 0;
282
- for (int i = 0; i < topk_length; ++i) {
283
- occurrences += (topk_ids[i] == expert_id);
284
- }
285
- expert_offsets[expert_id + 1] = occurrences;
286
- __syncthreads();
287
-
288
- if (threadIdx.x == 0) {
289
- int tot_offset = 0;
290
- expert_offsets[0] = 0;
291
- for (int i = 0; i < num_experts; ++i) {
292
- tot_offset += ceildiv(expert_offsets[i + 1], block_size) * block_size;
293
- expert_offsets[i + 1] = tot_offset;
294
- }
295
- }
296
- __syncthreads();
297
-}
298
-
299
-template <const int threads, // number of threads in a threadblock
300
- const int thread_m_blocks, // number of 16x16 blocks in the m
301
- // dimension (batchsize) of the
302
- // threadblock
303
- const int thread_n_blocks, // same for n dimension (output)
304
- const int thread_k_blocks, // same for k dimension (reduction)
305
- const int stages, // number of stages for the async global->shared
306
- // fetch pipeline
307
- const bool has_act_order, // whether act_order is enabled
308
- const int group_blocks = -1 // number of consecutive 16x16 blocks
309
- // with a separate quantization scale
310
- >
311
-__device__ inline void MarlinMoESingle(
312
- const int4* __restrict__ A, // fp16 input matrix of shape mxk
313
- const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
314
- int4* __restrict__ C, // fp16 output buffer of shape mxn
315
- const int* __restrict__ sorted_ids, // int32 sorted ids of experts
316
- const float* __restrict__ topk_weights, // float topk weights
317
- const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
318
- // (k/groupsize)xn
319
- const int* __restrict__ g_idx, // int32 group indices of shape k
320
- const int* __restrict__ expert_offsets,
321
- int num_groups, // number of scale groups per output channel
322
- int expert_idx, // idx of current expert
323
- int num_experts, // number of experts
324
- int topk, // topk parameter of moe
325
- int prob_m, // batch dimension m
326
- int prob_n, // output dimension n
327
- int prob_k, // reduction dimension k
328
- int tot_m, // total number of rows in A and C
329
- int* locks, // extra global storage for barrier synchronization
330
- bool replicate_input, // do we use the same input for each expert?
331
- bool apply_weights, // apply weights to output
332
- int current_m_block // current m block to start kernel computation from
333
-) {
334
- // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
335
- // better partitioning with less reductions
336
- int parallel = 1;
337
- if (prob_m > 16 * thread_m_blocks) {
338
- parallel = prob_m / (16 * thread_m_blocks);
339
- prob_m = 16 * thread_m_blocks;
340
- }
341
-
342
- int k_tiles = prob_k / 16 / thread_k_blocks;
343
- int n_tiles = prob_n / 16 / thread_n_blocks;
344
- int iters = ceildiv(k_tiles * n_tiles * parallel, gridDim.x);
345
-
346
- if constexpr (!has_act_order && group_blocks != -1) {
347
- if (group_blocks >= thread_k_blocks) {
348
- // Ensure that the number of tiles in each stripe is a multiple of the
349
- // groupsize; this avoids an annoying special case where a stripe starts
350
- // in the middle of group.
351
- iters = (group_blocks / thread_k_blocks) *
352
- ceildiv(iters, (group_blocks / thread_k_blocks));
353
- }
354
- }
355
-
356
- int slice_row = (iters * blockIdx.x) % k_tiles;
357
- int slice_col_par = (iters * blockIdx.x) / k_tiles;
358
- int slice_col = slice_col_par;
359
- int slice_iters; // number of threadblock tiles in the current slice
360
- int slice_count =
361
- 0; // total number of active threadblocks in the current slice
362
- int slice_idx; // index of threadblock in current slice; numbered bottom to
363
- // top
364
-
365
- // We can easily implement parallel problem execution by just remapping
366
- // indices and advancing global pointers
367
- if (slice_col_par >= n_tiles) {
368
- locks += (slice_col_par / n_tiles) * n_tiles;
369
- slice_col = slice_col_par % n_tiles;
370
- sorted_ids += (slice_col_par / n_tiles) * 16 * thread_m_blocks;
371
- }
372
-
373
- // Compute all information about the current slice which is required for
374
- // synchronization.
375
- auto init_slice = [&]() {
376
- slice_iters =
377
- iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
378
- if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
379
- if (slice_iters == 0) return;
380
- if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
381
- slice_count = 1;
382
- slice_idx = 0;
383
- int col_first = iters * ceildiv(k_tiles * slice_col_par, iters);
384
- if (col_first <= k_tiles * (slice_col_par + 1)) {
385
- int col_off = col_first - k_tiles * slice_col_par;
386
- slice_count = ceildiv(k_tiles - col_off, iters);
387
- if (col_off > 0) slice_count++;
388
- int delta_first = iters * blockIdx.x - col_first;
389
- if (delta_first < 0 || (col_off == 0 && delta_first == 0))
390
- slice_idx = slice_count - 1;
391
- else {
392
- slice_idx = slice_count - 1 - delta_first / iters;
393
- if (col_off > 0) slice_idx--;
394
- }
395
- }
396
- if (slice_col == n_tiles) {
397
- sorted_ids += 16 * thread_m_blocks;
398
- locks += n_tiles;
399
- slice_col = 0;
400
- }
401
- };
402
- init_slice();
403
-
404
- // A sizes/strides
405
-
406
- // stride of the A matrix in global memory
407
- int a_gl_stride = prob_k / 8;
408
- // stride of an A matrix tile in shared memory
409
- constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
410
- // delta between subsequent A tiles in global memory
411
- constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
412
- // between subsequent accesses within a tile
413
- int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
414
- // between shared memory writes
415
- constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
416
- // between shared memory tile reads
417
- constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
418
- // within a shared memory tile
419
- constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
420
- // overall size of a tile
421
- constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
422
- // number of shared write iterations for a tile
423
- constexpr int a_sh_wr_iters = ceildiv(a_sh_stage, a_sh_wr_delta);
424
-
425
- // B sizes/strides
426
- int b_gl_stride = 16 * prob_n / 32;
427
- constexpr int b_sh_stride = 32 * thread_n_blocks / 4;
428
- int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
429
- int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride);
430
- constexpr int b_sh_wr_delta = threads;
431
- constexpr int b_sh_rd_delta = threads;
432
- constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
433
- constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
434
-
435
- // Scale sizes/strides without act_order
436
- int s_gl_stride = prob_n / 8;
437
- constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
438
- constexpr int s_tb_groups = !has_act_order && group_blocks < thread_k_blocks
439
- ? thread_k_blocks / group_blocks
440
- : 1;
441
- constexpr int s_sh_stage = s_tb_groups * s_sh_stride;
442
- int s_gl_rd_delta = s_gl_stride;
443
- // Scale size/strides with act_order
444
- constexpr int tb_k = 16 * thread_k_blocks;
445
- constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0;
446
- // constexpr int act_s_row_stride = 1;
447
- // int act_s_col_stride = act_s_row_stride * num_groups;
448
- int act_s_col_stride = 1;
449
- int act_s_col_warp_stride = act_s_col_stride * 8;
450
- int tb_n_warps = thread_n_blocks / 4;
451
- int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
452
-
453
- constexpr int sorted_sh_stride = threads;
454
- constexpr int sorted_gl_stride = threads;
455
-
456
- // Global A read index of current thread.
457
- int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
458
- (threadIdx.x % a_gl_rd_delta_o);
459
- a_gl_rd += a_gl_rd_delta_o * slice_row;
460
- // Shared write index of current thread.
461
- int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
462
- (threadIdx.x % a_gl_rd_delta_o);
463
- // Shared read index.
464
- int a_sh_rd =
465
- a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
466
- a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
467
-
468
- int b_gl_rd =
469
- b_gl_stride * (threadIdx.x / b_sh_stride) + (threadIdx.x % b_sh_stride);
470
- b_gl_rd += b_sh_stride * slice_col;
471
- b_gl_rd += b_gl_rd_delta_o * slice_row;
472
- int b_sh_wr = threadIdx.x;
473
- int b_sh_rd = threadIdx.x;
474
-
475
- // For act_order
476
- constexpr int k_iter_size = tb_k / b_sh_wr_iters;
477
- int slice_k_start = tb_k * slice_row;
478
- int slice_k_finish = slice_k_start + tb_k * slice_iters;
479
- int slice_k_start_shared_fetch = slice_k_start;
480
- int slice_n_offset = act_s_col_tb_stride * slice_col;
481
-
482
- // No act_order
483
- int s_gl_rd;
484
- if constexpr (group_blocks == -1 || group_blocks == 0) {
485
- s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
486
- } else {
487
- s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) +
488
- s_sh_stride * slice_col + threadIdx.x;
489
- }
490
- int s_sh_wr = threadIdx.x;
491
- bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
492
-
493
- // We use a different scale layout for grouped and column-wise quantization as
494
- // we scale a `half2` tile in column-major layout in the former and in
495
- // row-major in the latter case.
496
- int s_sh_rd;
497
- if constexpr (group_blocks != -1)
498
- s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
499
- (threadIdx.x % 32) / 4;
500
- else
501
- s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) +
502
- (threadIdx.x % 32) % 4;
503
-
504
- int sh_first_group_id = -1;
505
- int sh_num_groups = -1;
506
- constexpr int sh_max_num_groups = 32;
507
-
508
- int shs_size;
509
- if constexpr (has_act_order)
510
- shs_size = sh_max_num_groups * s_sh_stride + threads;
511
- else
512
- shs_size = group_blocks > 0 ? stages * s_sh_stage : threads;
513
-
514
- extern __shared__ int4 sh[];
515
- // Shared memory storage for global fetch pipelines.
516
- int4* sh_a = sh;
517
- int4* sh_b = sh_a + (stages * a_sh_stage);
518
- int4* sh_g_idx = sh_b + (stages * b_sh_stage);
519
- int4* sh_s = sh_g_idx + (stages * g_idx_stage);
520
- int* sh_sorted = (int*)(sh_s + shs_size);
521
-
522
- // Precompute which thread should not read memory in which iterations; this is
523
- // needed if there are more threads than required for a certain tilesize or
524
- // when the batchsize is not a multiple of 16.
525
- bool a_sh_wr_pred[a_sh_wr_iters];
526
- #pragma unroll
527
- for (int i = 0; i < a_sh_wr_iters; i++) {
528
- int a_idx = a_sh_wr_delta * i + a_sh_wr;
529
- int row = a_idx / a_gl_rd_delta_o;
530
- if (row >= prob_m) {
531
- a_sh_wr_pred[i] = false;
532
- } else {
533
- a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
534
- }
535
- }
536
-
537
- // To ensure that writing and reading A tiles to/from shared memory, the
538
- // latter in fragment format, is fully bank conflict free, we need to use a
539
- // rather fancy XOR-based layout. The key here is that neither reads nor
540
- // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
541
- // same shared memory banks. Further, it seems (based on NSight-Compute) that
542
- // each warp must also write a consecutive memory segment?
543
- auto transform_a = [&](int i) {
544
- int row = i / a_gl_rd_delta_o;
545
- return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
546
- };
547
- // Since the computation of this remapping is non-trivial and, due to our main
548
- // loop unrolls, all shared memory accesses are static, we simply precompute
549
- // both transformed reads and writes.
550
- int a_sh_wr_trans[a_sh_wr_iters];
551
- #pragma unroll
552
- for (int i = 0; i < a_sh_wr_iters; i++)
553
- a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
554
- int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
555
- #pragma unroll
556
- for (int i = 0; i < b_sh_wr_iters; i++) {
557
- #pragma unroll
558
- for (int j = 0; j < thread_m_blocks; j++)
559
- a_sh_rd_trans[i][j] =
560
- transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
561
- }
562
-
563
- // Since B-accesses have non-constant stride they have to be computed at
564
- // runtime; we break dependencies between subsequent accesses with a tile by
565
- // maintining multiple pointers (we have enough registers), a tiny
566
- // optimization.
567
- const int4* B_ptr[b_sh_wr_iters];
568
- #pragma unroll
569
- for (int i = 0; i < b_sh_wr_iters; i++)
570
- B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
571
-
572
- // Register storage for double buffer of shared memory reads.
573
- FragA frag_a[2][thread_m_blocks];
574
- I4 frag_b_quant[2];
575
- FragC frag_c[thread_m_blocks][4][2];
576
- FragS frag_s[2][4]; // No act-order
577
- FragS act_frag_s[2][4][4]; // For act-order
578
-
579
- // Zero accumulators.
580
- auto zero_accums = [&]() {
581
- #pragma unroll
582
- for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
583
- reinterpret_cast<float*>(frag_c)[i] = 0;
584
- };
585
-
586
- auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
587
- int last_group_id) {
588
- sh_first_group_id = first_group_id;
589
- sh_num_groups = last_group_id - first_group_id + 1;
590
-
591
- if (sh_num_groups < sh_max_num_groups) {
592
- sh_num_groups = sh_max_num_groups;
593
- }
594
-
595
- if (sh_first_group_id + sh_num_groups > num_groups) {
596
- sh_num_groups = num_groups - sh_first_group_id;
597
- }
598
-
599
- int row_offset = first_group_id * s_gl_stride;
600
-
601
- if (is_async) {
602
- for (int i = 0; i < sh_num_groups; i++) {
603
- if (threadIdx.x < s_sh_stride) {
604
- cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
605
- &scales_ptr[row_offset + (i * s_gl_stride) +
606
- slice_n_offset + threadIdx.x]);
607
- }
608
- }
609
- } else {
610
- for (int i = 0; i < sh_num_groups; i++) {
611
- if (threadIdx.x < s_sh_stride) {
612
- sh_s[(i * s_sh_stride) + threadIdx.x] =
613
- scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
614
- threadIdx.x];
615
- }
616
- }
617
- }
618
- };
619
- // Asynchronously fetch the next A, B and s tile from global to the next
620
- // shared memory pipeline location.
621
- auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
622
- if (pred) {
623
- int4* sh_a_stage = sh_a + a_sh_stage * pipe;
624
- #pragma unroll
625
- for (int i = 0; i < a_sh_wr_iters; i++) {
626
- int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off;
627
- int row = a_idx / a_gl_stride;
628
- int sorted_row =
629
- replicate_input ? sorted_ids[row] / topk : sorted_ids[row];
630
- int new_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride;
631
- if (sorted_row < tot_m * (replicate_input ? 1 : topk) &&
632
- new_idx < a_gl_stride * tot_m * (replicate_input ? 1 : topk)) {
633
- cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[new_idx],
634
- a_sh_wr_pred[i]);
635
- }
636
- }
637
- int4* sh_b_stage = sh_b + b_sh_stage * pipe;
638
- #pragma unroll
639
- for (int i = 0; i < b_sh_wr_iters; i++) {
640
- cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr], B_ptr[i]);
641
- B_ptr[i] += b_gl_rd_delta_o;
642
- }
643
-
644
- if constexpr (has_act_order) {
645
- // Fetch g_idx thread-block portion
646
- int full_pipe = a_off;
647
- int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe;
648
- if (cur_k < prob_k && cur_k < slice_k_finish) {
649
- int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
650
-
651
- int4 const* cur_g_idx_stage_ptr =
652
- reinterpret_cast<int4 const*>(&g_idx[cur_k]);
653
-
654
- if (threadIdx.x < g_idx_stage) {
655
- cp_async4_pred(&sh_g_idx_stage[threadIdx.x],
656
- &cur_g_idx_stage_ptr[threadIdx.x]);
657
- }
658
- }
659
- } else {
660
- if constexpr (group_blocks != -1) {
661
- int4* sh_s_stage = sh_s + s_sh_stage * pipe;
662
-
663
- if constexpr (group_blocks >= thread_k_blocks) {
664
- // Only fetch scales if this tile starts a new group
665
- if (pipe % (group_blocks / thread_k_blocks) == 0) {
666
- if (s_sh_wr_pred) {
667
- cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]);
668
- }
669
- s_gl_rd += s_gl_rd_delta;
670
- }
671
- } else {
672
- for (int i = 0; i < s_tb_groups; i++) {
673
- if (s_sh_wr_pred) {
674
- cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr],
675
- &scales_ptr[s_gl_rd]);
676
- }
677
- s_gl_rd += s_gl_rd_delta;
678
- }
679
- }
680
- }
681
- }
682
- }
683
- // Insert a fence even when we are winding down the pipeline to ensure that
684
- // waiting is also correct at this point.
685
- cp_async_fence();
686
- };
687
-
688
- // TODO we are currently hitting illegal memory accesses when fetching
689
- // sorted_ids to shared data: fix this
690
- auto fetch_sorted_ids_to_shared = [&]() {
691
- const int mpt = ceildiv(prob_m, threads);
692
- for (int i = 0; i < mpt; i++) {
693
- if ((i * sorted_gl_stride) + threadIdx.x < prob_m) {
694
- sh_sorted[(i * sorted_sh_stride) + threadIdx.x] =
695
- sorted_ids[(i * sorted_gl_stride) + threadIdx.x];
696
- }
697
- }
698
- };
699
-
700
- // Wait until the next thread tile has been loaded to shared memory.
701
- auto wait_for_stage = [&]() {
702
- // We only have `stages - 2` active fetches since we are double buffering
703
- // and can only issue the next fetch when it is guaranteed that the previous
704
- // shared memory load is fully complete (as it may otherwise be
705
- // overwritten).
706
- cp_async_wait<stages - 2>();
707
- __syncthreads();
708
- };
709
-
710
- // Load the next sub-tile from the current location in the shared memory pipe
711
- // into the current register buffer.
712
- auto fetch_to_registers = [&](int k, int pipe) {
713
- int4* sh_a_stage = sh_a + a_sh_stage * pipe;
714
- #pragma unroll
715
- for (int i = 0; i < thread_m_blocks; i++)
716
- ldsm4(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
717
- int4* sh_b_stage = sh_b + b_sh_stage * pipe;
718
- frag_b_quant[k % 2] = *reinterpret_cast<I4*>(
719
- &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd]);
720
- };
721
-
722
- bool is_same_group[stages];
723
- int same_group_id[stages];
724
-
725
- auto init_same_group = [&](int pipe) {
726
- int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
727
- int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
728
-
729
- int group_id_1 = sh_g_idx_int_ptr[0];
730
- int group_id_2 = sh_g_idx_int_ptr[tb_k - 1];
731
-
732
- is_same_group[pipe] = group_id_1 == group_id_2;
733
- same_group_id[pipe] = group_id_1;
734
- };
735
-
736
- auto fetch_scales_to_registers = [&](int k, int full_pipe) {
737
- int pipe = full_pipe % stages;
738
-
739
- if constexpr (!has_act_order) {
740
- // No act-order case
741
- if constexpr (group_blocks != -1) {
742
- if constexpr (group_blocks >= thread_k_blocks) {
743
- int4* sh_s_stage =
744
- sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) *
745
- (pipe / (group_blocks / thread_k_blocks)));
746
- reinterpret_cast<int4*>(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd];
747
- } else {
748
- int warp_id = threadIdx.x / 32;
749
- int n_warps = thread_n_blocks / 4;
750
-
751
- int warp_row = warp_id / n_warps;
752
-
753
- int cur_k = warp_row * 16;
754
- cur_k += k_iter_size * (k % b_sh_wr_iters);
755
-
756
- int k_blocks = cur_k / 16;
757
- int cur_group_id = k_blocks / group_blocks;
758
-
759
- int4* sh_s_stage = sh_s + s_sh_stage * pipe;
760
-
761
- reinterpret_cast<int4*>(&frag_s[k % 2])[0] =
762
- sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride];
763
- }
764
- }
765
-
766
- return;
767
- }
768
-
769
- // Act-order case
770
-
771
- // Determine K of the "current" thread-block
772
- int cur_k = slice_k_start + tb_k * full_pipe;
773
- if (cur_k >= prob_k || cur_k >= slice_k_finish) {
774
- return;
775
- }
776
-
777
- // Reset (to current thread-block) since we read g_idx portion from the
778
- // shared memory
779
- cur_k = 0;
780
-
781
- // Progress to current iteration
782
- cur_k += k_iter_size * (k % b_sh_wr_iters);
783
-
784
- // Determine "position" inside the thread-block (based on warp and
785
- // thread-id)
786
- int warp_id = threadIdx.x / 32;
787
- int n_warps =
788
- thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N
789
-
790
- int warp_row = warp_id / n_warps;
791
- int warp_col = warp_id % n_warps;
792
-
793
- cur_k += warp_row * 16;
794
-
795
- int th_id = threadIdx.x % 32;
796
- cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix
797
-
798
- int s_col_shift =
799
- /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) +
800
- (th_id / 4) * act_s_col_stride;
801
-
802
- if (is_same_group[pipe]) {
803
- if (k % 2 == 0) {
804
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
805
- sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride +
806
- s_col_shift];
807
- } else {
808
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0]))) =
809
- *(reinterpret_cast<int4*>(&(act_frag_s[(k - 1) % 2][0][0])));
810
- }
811
-
812
- for (int i = 1; i < 4; i++) {
813
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
814
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][0][0])));
815
- }
816
- return;
817
- }
818
-
819
- int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe;
820
- int* sh_g_idx_int_ptr = reinterpret_cast<int*>(sh_g_idx_stage);
821
-
822
- constexpr int k_frag_offsets[4] = {0, 1, 8,
823
- 9}; // Tensor core offsets per thread
824
-
825
- #pragma unroll
826
- for (int i = 0; i < 4; i++) {
827
- int actual_k = cur_k + k_frag_offsets[i];
828
-
829
- int group_id = sh_g_idx_int_ptr[actual_k];
830
- int rel_group_id = group_id - sh_first_group_id;
831
-
832
- *(reinterpret_cast<int4*>(&(act_frag_s[k % 2][i][0]))) =
833
- sh_s[rel_group_id * s_sh_stride + s_col_shift];
834
- }
835
- };
836
-
837
- // Execute the actual tensor core matmul of a sub-tile.
838
- auto matmul = [&](int k) {
839
- // We have the m dimension as the inner loop in order to encourage overlapping
840
- // dequantization and matmul operations.
841
- #pragma unroll
842
- for (int j = 0; j < 4; j++) {
843
- int b_quant = frag_b_quant[k % 2][j];
844
- int b_quant_shift = b_quant >> 8;
845
-
846
- FragB frag_b0 = dequant(b_quant);
847
-
848
- // Apply scale to frag_b0
849
- if constexpr (has_act_order) {
850
- scale4(frag_b0, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
851
- act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 0);
852
- } else {
853
- if constexpr (group_blocks != -1) {
854
- scale(frag_b0, frag_s[k % 2][j], 0);
855
- }
856
- }
857
-
858
- FragB frag_b1 = dequant(b_quant_shift);
859
-
860
- // Apply scale to frag_b1
861
- if constexpr (has_act_order) {
862
- scale4(frag_b1, act_frag_s[k % 2][0][j], act_frag_s[k % 2][1][j],
863
- act_frag_s[k % 2][2][j], act_frag_s[k % 2][3][j], 1);
864
-
865
- } else {
866
- if constexpr (group_blocks != -1) {
867
- scale(frag_b1, frag_s[k % 2][j], 1);
868
- }
869
- }
870
-
871
- #pragma unroll
872
- for (int i = 0; i < thread_m_blocks; i++) {
873
- mma(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
874
- mma(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
875
- }
876
- }
877
- };
878
-
879
- // Since we slice across the k dimension of a tile in order to increase the
880
- // number of warps while keeping the n dimension of a tile reasonable, we have
881
- // multiple warps that accumulate their partial sums of the same output
882
- // location; which we have to reduce over in the end. We do in shared memory.
883
- auto thread_block_reduce = [&]() {
884
- constexpr int red_off = threads / b_sh_stride / 2;
885
- if (red_off >= 1) {
886
- int red_idx = threadIdx.x / b_sh_stride;
887
- constexpr int red_sh_stride = b_sh_stride * 4 * 2;
888
- constexpr int red_sh_delta = b_sh_stride;
889
- int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride) +
890
- (threadIdx.x % b_sh_stride);
891
-
892
- // Parallel logarithmic shared memory reduction. We make sure to avoid any
893
- // unnecessary read or write iterations, e.g., for two warps we write only
894
- // once by warp 1 and read only once by warp 0.
895
-
896
- #pragma unroll
897
- for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
898
- #pragma unroll
899
- for (int i = red_off; i > 0; i /= 2) {
900
- if (i <= red_idx && red_idx < 2 * i) {
901
- #pragma unroll
902
- for (int j = 0; j < 4 * 2; j++) {
903
- int red_sh_wr =
904
- red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
905
- if (i < red_off) {
906
- float* c_rd =
907
- reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
908
- float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
909
- #pragma unroll
910
- for (int k = 0; k < 4; k++)
911
- reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
912
- c_rd[k] + c_wr[k];
913
- }
914
- sh[red_sh_wr] =
915
- reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
916
- }
917
- }
918
- __syncthreads();
919
- }
920
- if (red_idx == 0) {
921
- #pragma unroll
922
- for (int i = 0; i < 4 * 2; i++) {
923
- float* c_rd =
924
- reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
925
- #pragma unroll
926
- for (int j = 0; j < 4; j++)
927
- reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
928
- c_rd[j];
929
- }
930
- }
931
- __syncthreads();
932
- }
933
- }
934
- };
935
-
936
- // Since multiple threadblocks may process parts of the same column slice, we
937
- // finally have to globally reduce over the results. As the striped
938
- // partitioning minimizes the number of such reductions and our outputs are
939
- // usually rather small, we perform this reduction serially in L2 cache.
940
- auto global_reduce = [&](bool first = false, bool last = false) {
941
- // We are very careful here to reduce directly in the output buffer to
942
- // maximize L2 cache utilization in this step. To do this, we write out
943
- // results in FP16 (but still reduce with FP32 compute).
944
- constexpr int active_threads = 32 * thread_n_blocks / 4;
945
- if (threadIdx.x < active_threads) {
946
- int c_gl_stride = prob_n / 8;
947
- int c_gl_wr_delta_o = 8 * c_gl_stride;
948
- int c_gl_wr_delta_i = 4 * (active_threads / 32);
949
- int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
950
- 4 * (threadIdx.x / 32) + threadIdx.x % 4;
951
- c_gl_wr += (2 * thread_n_blocks) * slice_col;
952
- constexpr int c_sh_wr_delta = active_threads;
953
- int c_sh_wr = threadIdx.x;
954
-
955
- int row = (threadIdx.x % 32) / 4;
956
-
957
- if (!first) {
958
- // Interestingly, doing direct global accesses here really seems to mess up
959
- // the compiler and lead to slowdowns, hence we also use async-copies even
960
- // though these fetches are not actually asynchronous.
961
- #pragma unroll
962
- for (int i = 0; i < thread_m_blocks * 4; i++) {
963
- int c_idx =
964
- c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
965
- int sorted_row = sorted_ids[c_idx / c_gl_stride];
966
- int new_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride;
967
- cp_async4_pred(&sh[c_sh_wr + c_sh_wr_delta * i], &C[new_idx],
968
- sorted_row < tot_m * topk &&
969
- (8 * (i / 2) + row < prob_m &&
970
- (i < (thread_m_blocks - 1) * 4 ||
971
- sorted_ids[8 * (i / 2) + row] < tot_m * topk)));
972
- }
973
- cp_async_fence();
974
- cp_async_wait<0>();
975
- }
976
-
977
- #pragma unroll
978
- for (int i = 0; i < thread_m_blocks * 4; i++) {
979
- if (8 * (i / 2) + row < prob_m &&
980
- (i < (thread_m_blocks - 1) * 4 ||
981
- sorted_ids[8 * (i / 2) + row] < tot_m * topk)) {
982
- if (!first) {
983
- int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
984
- #pragma unroll
985
- for (int j = 0; j < 2 * 4; j++) {
986
- reinterpret_cast<float*>(
987
- &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
988
- __half2float(reinterpret_cast<__half*>(&c_red)[j]);
989
- }
990
- }
991
- if (!last) {
992
- int4 c;
993
- #pragma unroll
994
- for (int j = 0; j < 2 * 4; j++) {
995
- reinterpret_cast<__half*>(&c)[j] =
996
- __float2half(reinterpret_cast<float*>(
997
- &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
998
- }
999
- int c_idx =
1000
- c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2);
1001
- int row = sorted_ids[c_idx / c_gl_stride];
1002
- if (row < tot_m * topk) {
1003
- int new_idx = row * c_gl_stride + c_idx % c_gl_stride;
1004
- C[new_idx] = c;
1005
- }
1006
- }
1007
- }
1008
- }
1009
- }
1010
- };
1011
-
1012
- // Write out the reduce final result in the correct layout. We only actually
1013
- // reshuffle matrix fragments in this step, the reduction above is performed
1014
- // in fragment layout.
1015
- auto write_result = [&]() {
1016
- int c_gl_stride = prob_n / 8;
1017
- constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
1018
- int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
1019
- constexpr int c_sh_rd_delta =
1020
- c_sh_stride * (threads / (2 * thread_n_blocks));
1021
-
1022
- int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1023
- (threadIdx.x % (2 * thread_n_blocks));
1024
- c_gl_wr += (2 * thread_n_blocks) * slice_col;
1025
- int c_sh_wr =
1026
- (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
1027
- c_sh_wr += 32 * (threadIdx.x / 32);
1028
- int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
1029
- (threadIdx.x % (2 * thread_n_blocks));
1030
-
1031
- int c_gl_wr_end = c_gl_stride * prob_m;
1032
-
1033
- // We first reorder in shared memory to guarantee the most efficient final
1034
- // global write patterns
1035
- auto write = [&](int idx, float c0, float c1, FragS& s) {
1036
- half2 res = __halves2half2(__float2half(c0), __float2half(c1));
1037
-
1038
- // For per-column quantization we finally apply the scale here
1039
- if constexpr (!has_act_order && group_blocks == -1) {
1040
- res = __hmul2(res, s[0]);
1041
- }
1042
-
1043
- ((half2*)sh)[idx] = res;
1044
- };
1045
- if (threadIdx.x / 32 < thread_n_blocks / 4) {
1046
- #pragma unroll
1047
- for (int i = 0; i < thread_m_blocks; i++) {
1048
- #pragma unroll
1049
- for (int j = 0; j < 4; j++) {
1050
- int wr = c_sh_wr + 8 * j;
1051
- write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
1052
- frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
1053
- write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
1054
- frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
1055
- write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
1056
- frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
1057
- write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
1058
- frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
1059
- }
1060
- c_sh_wr += 16 * (4 * c_sh_stride);
1061
- }
1062
- }
1063
- __syncthreads();
1064
-
1065
- #pragma unroll
1066
- for (int i = 0;
1067
- i < ceildiv(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
1068
- i++) {
1069
- if (c_gl_wr < c_gl_wr_end) {
1070
- int row = sorted_ids[c_gl_wr / c_gl_stride];
1071
- if (row < tot_m * topk) {
1072
- int off = row * c_gl_stride + c_gl_wr % c_gl_stride;
1073
- if (!apply_weights) {
1074
- C[off] = sh[c_sh_rd];
1075
- } else {
1076
- __half* ctrg = reinterpret_cast<__half*>(&C[off]);
1077
- __half* csrc = reinterpret_cast<__half*>(&sh[c_sh_rd]);
1078
- for (int j = 0; j < 8; ++j) {
1079
- ctrg[j] = __float2half(topk_weights[row] * __half2float(csrc[j]));
1080
- }
1081
- }
1082
- c_gl_wr += c_gl_wr_delta;
1083
- c_sh_rd += c_sh_rd_delta;
1084
- }
1085
- }
1086
- }
1087
- };
1088
-
1089
- // Start global fetch and register load pipelines.
1090
- auto start_pipes = [&]() {
1091
- // TODO re-enable after fixing this function
1092
- // fetch_sorted_ids_to_shared();
1093
- __syncthreads();
1094
-
1095
- #pragma unroll
1096
- for (int i = 0; i < stages - 1; i++) {
1097
- if (has_act_order && i == 0) {
1098
- int last_g_idx = slice_k_start + stages * tb_k * 2;
1099
- if (last_g_idx >= prob_k) {
1100
- last_g_idx = prob_k - 1;
1101
- }
1102
- fetch_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]);
1103
- }
1104
- fetch_to_shared(i, i, i < slice_iters);
1105
- }
1106
-
1107
- zero_accums();
1108
- wait_for_stage();
1109
- init_same_group(0);
1110
- fetch_to_registers(0, 0);
1111
- fetch_scales_to_registers(0, 0);
1112
- a_gl_rd += a_gl_rd_delta_o * (stages - 1);
1113
- slice_k_start_shared_fetch += tb_k * (stages - 1);
1114
- };
1115
- if (slice_iters) {
1116
- start_pipes();
1117
- }
1118
-
1119
- // Main loop.
1120
- while (slice_iters) {
1121
- // We unroll over both the global fetch and the register load pipeline to
1122
- // ensure all shared memory accesses are static. Note that both pipelines
1123
- // have even length meaning that the next iteration will always start at
1124
- // index 0.
1125
- #pragma unroll
1126
- for (int pipe = 0; pipe < stages;) {
1127
- #pragma unroll
1128
- for (int k = 0; k < b_sh_wr_iters; k++) {
1129
- fetch_to_registers(k + 1, pipe % stages);
1130
- fetch_scales_to_registers(k + 1, pipe);
1131
- if (k == b_sh_wr_iters - 2) {
1132
- fetch_to_shared((pipe + stages - 1) % stages, pipe,
1133
- slice_iters >= stages);
1134
- pipe++;
1135
- wait_for_stage();
1136
- init_same_group(pipe % stages);
1137
- }
1138
- matmul(k);
1139
- }
1140
- slice_iters--;
1141
- if (slice_iters == 0) {
1142
- break;
1143
- }
1144
- }
1145
-
1146
- a_gl_rd += a_gl_rd_delta_o * stages;
1147
- slice_k_start += tb_k * stages;
1148
- slice_k_start_shared_fetch += tb_k * stages;
1149
-
1150
- if constexpr (has_act_order) {
1151
- int first_group_id = g_idx[slice_k_start];
1152
- int last_g_idx = slice_k_start + stages * tb_k * 2;
1153
- if (last_g_idx >= prob_k) {
1154
- last_g_idx = prob_k - 1;
1155
- }
1156
- int last_group_id = g_idx[last_g_idx];
1157
- if (last_group_id >= sh_first_group_id + sh_num_groups) {
1158
- fetch_scales_to_shared(false, first_group_id, last_group_id);
1159
- __syncthreads();
1160
- }
1161
- }
1162
-
1163
- // Process results and, if necessary, proceed to the next column slice.
1164
- // While this pattern may not be the most readable, other ways of writing
1165
- // the loop seemed to noticeably worse performance after compilation.
1166
- if (slice_iters == 0) {
1167
- cp_async_wait<0>();
1168
- bool last = slice_idx == slice_count - 1;
1169
- // For per-column scales, we only fetch them here in the final step before
1170
- // write-out
1171
- if constexpr (!has_act_order && group_blocks == -1) {
1172
- if (last) {
1173
- if (s_sh_wr_pred) {
1174
- cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
1175
- }
1176
- cp_async_fence();
1177
- }
1178
- }
1179
-
1180
- thread_block_reduce();
1181
- if constexpr (!has_act_order && group_blocks == -1) {
1182
- if (last) {
1183
- cp_async_wait<0>();
1184
- __syncthreads();
1185
- if (threadIdx.x / 32 < thread_n_blocks / 4) {
1186
- reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
1187
- reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
1188
- }
1189
- }
1190
- }
1191
- if (slice_count > 1) { // only globally reduce if there is more than one
1192
- // block in a slice
1193
- barrier_acquire(&locks[slice_col], slice_idx);
1194
- global_reduce(slice_idx == 0, last);
1195
- barrier_release(&locks[slice_col], last);
1196
- }
1197
- if (last) // only the last block in a slice actually writes the result
1198
- write_result();
1199
- slice_row = 0;
1200
- slice_col_par++;
1201
- slice_col++;
1202
- init_slice();
1203
- if (slice_iters) {
1204
- a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
1205
- (threadIdx.x % a_gl_rd_delta_o);
1206
- #pragma unroll
1207
- for (int i = 0; i < b_sh_wr_iters; i++)
1208
- B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
1209
- if (slice_col == 0) {
1210
- #pragma unroll
1211
- for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
1212
- }
1213
-
1214
- // Update slice k/n for scales loading
1215
- if constexpr (has_act_order) {
1216
- slice_k_start = tb_k * slice_row;
1217
- slice_k_finish = slice_k_start + tb_k * slice_iters;
1218
- slice_k_start_shared_fetch = slice_k_start;
1219
- slice_n_offset = act_s_col_tb_stride * slice_col;
1220
-
1221
- } else {
1222
- s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
1223
- }
1224
- start_pipes();
1225
- }
1226
- }
1227
- }
1228
-}
1229
-
1230
-template <const int threads, // number of threads in a threadblock
1231
- const int thread_m_blocks, // number of 16x16 blocks in the m
1232
- // dimension (batchsize) of the
1233
- // threadblock
1234
- const int thread_n_blocks, // same for n dimension (output)
1235
- const int thread_k_blocks, // same for k dimension (reduction)
1236
- const int stages, // number of stages for the async global->shared
1237
- // fetch pipeline
1238
- const bool has_act_order, // whether act_order is enabled
1239
- const int group_blocks = -1 // number of consecutive 16x16 blocks
1240
- // with a separate quantization scale
1241
- >
1242
-__global__ void MarlinMoE(
1243
- const int4* __restrict__ A, // fp16 input matrix of shape mxk
1244
- const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1245
- int4* __restrict__ C, // fp16 output buffer of shape mxn
1246
- const int* __restrict__ sorted_ids_base, // int32 sorted ids of experts
1247
- const float* __restrict__ topk_weights, // float topk weights
1248
- const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1249
- // (k/groupsize)xn
1250
- const int* __restrict__ g_idx, // int32 group indices of shape k
1251
- const int* __restrict__ expert_offsets,
1252
- int num_groups, // number of scale groups per output channel
1253
- int expert_idx, // idx of current expert
1254
- int num_experts, // number of experts
1255
- int topk, // topk parameter of moe
1256
- int prob_m, // batch dimension m
1257
- int prob_n, // output dimension n
1258
- int prob_k, // reduction dimension k
1259
- int tot_m, // total number of rows in A and C
1260
- int* locks, // extra global storage for barrier synchronization
1261
- bool replicate_input, // do we use the same input for each expert?
1262
- bool apply_weights, // apply weights to output
1263
- int current_m_block, // current m block to start kernel computation from
1264
- int max_par // maximum parallelism
1265
-) {
1266
- int m_block_ctr = current_m_block;
1267
-
1268
- const int* sorted_ids_expert =
1269
- sorted_ids_base + expert_offsets[expert_idx] + m_block_ctr * 4 * max_par;
1270
- int tot_its = expert_offsets[expert_idx + 1] - expert_offsets[expert_idx];
1271
- if (tot_its == 0) {
1272
- return;
1273
- }
1274
- int tot_m_blocks = ceildiv(tot_its, 16);
1275
- int pad = 16 * tot_m_blocks - tot_its;
1276
-
1277
- if (m_block_ctr >= tot_m_blocks) {
1278
- return;
1279
- }
1280
-
1281
- int max_block = tot_m_blocks - m_block_ctr;
1282
- prob_m = tot_its - 16 * m_block_ctr;
1283
-
1284
- int par = 1;
1285
- if (max_block > 4) {
1286
- // Note that parallel > 1 currently only works for inputs without any
1287
- // padding
1288
- par = (16 * max_block - pad) / 64;
1289
- par = min((16 * max_block - pad) / 64, max_par);
1290
- prob_m = 64 * par;
1291
- m_block_ctr += 4 * (par - 1);
1292
- max_block = 4;
1293
- }
1294
-
1295
- if (max_block == 1) {
1296
- MarlinMoESingle<threads, 1, thread_n_blocks, thread_k_blocks, stages,
1297
- has_act_order, group_blocks>(
1298
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1299
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1300
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1301
- current_m_block);
1302
- } else if (max_block == 2) {
1303
- MarlinMoESingle<threads, 2, thread_n_blocks, thread_k_blocks, stages,
1304
- has_act_order, group_blocks>(
1305
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1306
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1307
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1308
- current_m_block);
1309
- } else if (max_block == 3) {
1310
- MarlinMoESingle<threads, 3, thread_n_blocks, thread_k_blocks, stages,
1311
- has_act_order, group_blocks>(
1312
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1313
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1314
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1315
- current_m_block);
1316
- } else {
1317
- MarlinMoESingle<threads, 4, thread_n_blocks, thread_k_blocks, stages,
1318
- has_act_order, group_blocks>(
1319
- A, B, C, sorted_ids_expert, topk_weights, scales_ptr, g_idx,
1320
- expert_offsets, num_groups, expert_idx, num_experts, topk, prob_m,
1321
- prob_n, prob_k, tot_m, locks, replicate_input, apply_weights,
1322
- current_m_block);
1323
- }
1324
-}
1325
-
1326
-#else
1327
-
1328
-__global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,
1329
- int const* __restrict__ perm_int_ptr,
1330
- int4* __restrict__ out_int4_ptr, int size_m,
1331
- int size_k, int block_rows) {
1332
- // Marlin is not implemented yet for SM < 8.0
1333
- assert(false);
1334
- return;
1335
-}
1336
-
1337
-__global__ void compute_expert_offsets(int const* __restrict__ topk_ids,
1338
- int* __restrict__ expert_offsets,
1339
- int topk_length, int block_size) {
1340
- // Marlin is not implemented yet for SM < 8.0
1341
- assert(false);
1342
- return;
1343
-}
1344
-
1345
-template <const int threads, // number of threads in a threadblock
1346
- const int thread_m_blocks, // number of 16x16 blocks in the m
1347
- // dimension (batchsize) of the
1348
- // threadblock
1349
- const int thread_n_blocks, // same for n dimension (output)
1350
- const int thread_k_blocks, // same for k dimension (reduction)
1351
- const int stages, // number of stages for the async global->shared
1352
- // fetch pipeline
1353
- const bool has_act_order, // whether act_order is enabled
1354
- const int group_blocks = -1 // number of consecutive 16x16 blocks
1355
- // with a separate quantization scale
1356
- >
1357
-__global__ void MarlinMoE(
1358
- const int4* __restrict__ A, // fp16 input matrix of shape mxk
1359
- const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
1360
- int4* __restrict__ C, // fp16 output buffer of shape mxn
1361
- const int* __restrict__ sorted_ids, // int32 sorted ids of experts
1362
- const float* __restrict__ topk_weights, // float topk weights
1363
- const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
1364
- // (k/groupsize)xn
1365
- const int* __restrict__ g_idx, // int32 group indices of shape k
1366
- const int* __restrict__ expert_offsets,
1367
- int num_groups, // number of scale groups per output channel
1368
- int expert_idx, // idx of current expert
1369
- int num_experts, // number of experts
1370
- int topk, // topk parameter of moe
1371
- int prob_m, // batch dimension m
1372
- int prob_n, // output dimension n
1373
- int prob_k, // reduction dimension k
1374
- int tot_m, // total number of rows in A and C
1375
- int* locks, // extra global storage for barrier synchronization
1376
- bool replicate_input, // do we use the same input for each expert?
1377
- bool apply_weights, // apply weights to output
1378
- int current_m_block, // current m block to start kernel computation from
1379
- int max_par // maximum parallelism
1380
-) {
1381
- // Marlin is not implemented yet for SM < 8.0
1382
- assert(false);
1383
- return;
1384
-}
1385
-
1386
-#endif
1387
-
1388
-// 8 warps are a good choice since every SM has 4 schedulers and having more
1389
-// than 1 warp per schedule allows some more latency hiding. At the same time,
1390
-// we want relatively few warps to have many registers per warp and small tiles.
1391
-const int USER_THREADS =
1392
- 256; // Note: This is only used with user-provided thread_k/n
1393
-const int STAGES = 4; // 4 pipeline stages fit into shared memory
1394
-// const int SHARED_MEM =
1395
-// 96 * 1024; // max shared memory on compute capability 8.6 (< 8.0)
1396
-
1397
-static constexpr int min_thread_n = 64;
1398
-static constexpr int min_thread_k = 64;
1399
-
1400
-#define __CALL_IF_MOE(THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1401
- HAS_ACT_ORDER, GROUP_BLOCKS, NUM_THREADS) \
1402
- else if (thread_m_blocks == THREAD_M_BLOCKS && \
1403
- thread_n_blocks == THREAD_N_BLOCKS && \
1404
- thread_k_blocks == THREAD_K_BLOCKS && \
1405
- has_act_order == HAS_ACT_ORDER && group_blocks == GROUP_BLOCKS && \
1406
- num_threads == NUM_THREADS) { \
1407
- cudaFuncSetAttribute( \
1408
- MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
1409
- THREAD_K_BLOCKS, STAGES, HAS_ACT_ORDER, GROUP_BLOCKS>, \
1410
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
1411
- MarlinMoE<NUM_THREADS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, \
1412
- STAGES, HAS_ACT_ORDER, GROUP_BLOCKS> \
1413
- <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
1414
- A_ptr, B_ptr, C_ptr, sorted_ids_ptr, topk_weights_ptr, s_ptr, \
1415
- g_idx_ptr, expert_offsets_ptr, num_groups, expert_idx, \
1416
- num_experts, topk, prob_m, prob_n, prob_k, tot_m, locks, \
1417
- replicate_input, apply_weights, m_block, max_par); \
1418
- }
1419
-
1420
-typedef struct {
1421
- int thread_k;
1422
- int thread_n;
1423
- int num_threads;
1424
-} thread_config_t;
1425
-
1426
-thread_config_t small_batch_thread_configs[] = {
1427
- // Ordered by priority
1428
-
1429
- // thread_k, thread_n, num_threads
1430
- {128, 128, 256}, // Default
1431
- {128, 64, 128}, // Reduce N 2X, same K
1432
- {64, 256, 256}, // Reduce K 2X, increase N 2X
1433
- {64, 128, 128}, // Reduce K 2X, same N
1434
-};
1435
-
1436
-thread_config_t large_batch_thread_configs[] = {
1437
- // Ordered by priority
1438
-
1439
- // thread_k, thread_n, num_threads
1440
- {64, 256, 256}, // Default
1441
- {128, 128, 256}, // Reduce N 2X, increase K 2X
1442
- {64, 128, 128}, // Reduce N 2X, same K
1443
- {128, 64, 128}, // Reduce N 4X, increase K 2X
1444
-};
1445
-
1446
-bool is_valid_config(thread_config_t const& th_config, int prob_m, int prob_n,
1447
- int prob_k) {
1448
- // Sanity
1449
- if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1450
- th_config.num_threads == -1) {
1451
- return false;
1452
- }
1453
-
1454
- // Verify K/N are divisible by thread K/N
1455
- if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) {
1456
- return false;
1457
- }
1458
-
1459
- // thread_k can be only 128 or 64 (because it must be less than groupsize
1460
- // which is 128)
1461
- if (th_config.thread_k != 128 && th_config.thread_k != 64) {
1462
- return false;
1463
- }
1464
-
1465
- // Verify min for thread K/N
1466
- if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) {
1467
- return false;
1468
- }
1469
-
1470
- // num_threads must be at least 128 (= 4 warps)
1471
- if (th_config.num_threads < 128) {
1472
- return false;
1473
- }
1474
-
1475
- return true;
1476
-}
1477
-
1478
-thread_config_t determine_thread_config(int prob_m, int prob_n, int prob_k) {
1479
- if (prob_m <= 16) {
1480
- for (auto th_config : small_batch_thread_configs) {
1481
- if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1482
- return th_config;
1483
- }
1484
- }
1485
-
1486
- } else {
1487
- for (auto th_config : large_batch_thread_configs) {
1488
- if (is_valid_config(th_config, prob_m, prob_n, prob_k)) {
1489
- return th_config;
1490
- }
1491
- }
1492
- }
1493
-
1494
- return thread_config_t{-1, -1, -1};
1495
-}
1496
-
1497
-#define CALL_IF_MOE(N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1498
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1499
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1500
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1501
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS) \
1502
- \
1503
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1504
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1505
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1506
- __CALL_IF_MOE(1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1507
- \
1508
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1509
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1510
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1511
- __CALL_IF_MOE(2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1512
- \
1513
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1514
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1515
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1516
- __CALL_IF_MOE(3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS) \
1517
- \
1518
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS) \
1519
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS) \
1520
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS) \
1521
- __CALL_IF_MOE(4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS)
1522
-
1523
-void marlin_mm_moe_f16i4(const void* A, const void* B, void* C,
1524
- const void* sorted_ids, const void* topk_weights,
1525
- const void* topk_ids, const void* s, const void* g_idx,
1526
- const void* perm, void* a_tmp, void* expert_offsets,
1527
- int prob_m, int prob_n, int prob_k, void* workspace,
1528
- bool has_act_order, bool is_k_full, int num_groups,
1529
- int group_size, int num_experts, int topk,
1530
- int moe_block_size, int dev, cudaStream_t stream,
1531
- int thread_k, int thread_n, int sms, int max_par,
1532
- bool replicate_input, bool apply_weights) {
1533
- TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1534
- ", ", prob_n, ", ", prob_k, "]");
1535
-
1536
- if (sms == -1) {
1537
- cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1538
- }
1539
-
1540
- // Set thread config
1541
- thread_config_t th_config;
1542
- if (thread_k != -1 && thread_n != -1) {
1543
- // User-defined config
1544
- th_config = thread_config_t{thread_k, thread_n, USER_THREADS};
1545
- } else {
1546
- // Auto config
1547
- th_config = determine_thread_config(prob_m, prob_n, prob_k);
1548
- }
1549
-
1550
- TORCH_CHECK(is_valid_config(th_config, prob_m, prob_n, prob_k),
1551
- "Invalid thread config: thread_k = " + str(th_config.thread_k) +
1552
- ", thread_n = " + str(th_config.thread_n) +
1553
- ", num_threads = " + str(th_config.num_threads) +
1554
- " for MKN = [" + str(prob_m) + ", " + str(prob_k) + ", " +
1555
- str(prob_n) + "]");
1556
-
1557
- int num_threads = th_config.num_threads;
1558
- thread_k = th_config.thread_k;
1559
- thread_n = th_config.thread_n;
1560
-
1561
- int thread_k_blocks = thread_k / 16;
1562
- int thread_n_blocks = thread_n / 16;
1563
-
1564
- int blocks = sms;
1565
-
1566
- TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
1567
- " is not divisible by thread_n = ", thread_n);
1568
- TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
1569
- " is not divisible by thread_k = ", thread_k);
1570
-
1571
- int group_blocks = 0;
1572
- if (has_act_order) {
1573
- if (is_k_full) {
1574
- TORCH_CHECK(group_size != -1);
1575
- group_blocks = group_size / 16;
1576
- TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1577
- " is not divisible by group_blocks = ", group_blocks);
1578
- } else {
1579
- TORCH_CHECK(group_size == 0);
1580
- group_blocks = 0;
1581
- }
1582
-
1583
- } else {
1584
- if (group_size == -1) {
1585
- group_blocks = -1;
1586
- } else {
1587
- group_blocks = group_size / 16;
1588
- TORCH_CHECK(prob_k % group_blocks == 0, "prob_k = ", prob_k,
1589
- " is not divisible by group_blocks = ", group_blocks);
1590
- }
1591
- }
1592
-
1593
- int max_shared_mem = 0;
1594
- cudaDeviceGetAttribute(&max_shared_mem,
1595
- cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1596
- TORCH_CHECK(max_shared_mem > 0);
1597
-
1598
- int tot_m = prob_m;
1599
-
1600
- const int* topk_ids_ptr = (const int*)topk_ids;
1601
- int* expert_offsets_ptr = (int*)expert_offsets;
1602
- compute_expert_offsets<<<1, num_experts, 0, stream>>>(
1603
- topk_ids_ptr, expert_offsets_ptr, tot_m * topk, moe_block_size);
1604
-
1605
- bool do_permute_a = has_act_order;
1606
-
1607
- // If we have a full K, then we can run the non-act-order version of Marlin
1608
- // (since the weight rows are reordered by increasing group ids, and by
1609
- // having a full K, we have full original groups)
1610
- if (is_k_full) {
1611
- has_act_order = false;
1612
- }
1613
-
1614
- for (int expert_idx = 0; expert_idx < num_experts; ++expert_idx) {
1615
- const int4* A_ptr = (const int4*)A;
1616
- int4* a_tmp_ptr = (int4*)a_tmp;
1617
- const int4* B_ptr = (const int4*)B + (prob_n * prob_k / 32) * expert_idx;
1618
- int4* C_ptr = (int4*)C;
1619
- const float* topk_weights_ptr = (const float*)topk_weights;
1620
- const int* sorted_ids_ptr = (const int*)sorted_ids;
1621
- const int4* s_ptr =
1622
- (const int4*)s +
1623
- (((group_size == -1 || group_size == 0) ? 1 : prob_k / group_size) *
1624
- prob_n / 8) *
1625
- expert_idx;
1626
- const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
1627
- const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
1628
- int* locks = (int*)workspace;
1629
-
1630
- if (do_permute_a) {
1631
- // Permute A columns
1632
- int topk_rows = replicate_input ? tot_m : tot_m * topk;
1633
- int block_rows = ceildiv(topk_rows, blocks);
1634
- permute_cols_kernel<<<blocks, num_threads, 0, stream>>>(
1635
- A_ptr, perm_ptr, a_tmp_ptr, topk_rows, prob_k, block_rows);
1636
- A_ptr = a_tmp_ptr;
1637
- }
1638
-
1639
- int max_m_blocks = ceildiv(tot_m, 16);
1640
- for (int m_block = 0; m_block < max_m_blocks; m_block += 16) {
1641
- // Define kernel configurations
1642
-
1643
- // make it max possible value
1644
- int thread_m_blocks = 4;
1645
-
1646
- if (false) {
1647
- }
1648
- CALL_IF_MOE(16, 4, 256)
1649
- CALL_IF_MOE(8, 8, 256)
1650
- CALL_IF_MOE(8, 4, 128)
1651
- CALL_IF_MOE(4, 8, 128)
1652
- else {
1653
- TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1654
- str(prob_n) + ", " + str(prob_k) + "]" +
1655
- ", has_act_order = " + str(has_act_order) +
1656
- ", num_groups = " + str(num_groups) +
1657
- ", group_size = " + str(group_size) +
1658
- ", thread_m_blocks = " + str(thread_m_blocks) +
1659
- ", thread_n_blocks = " + str(thread_n_blocks) +
1660
- ", thread_k_blocks = " + str(thread_k_blocks));
1661
- }
1662
- }
1663
- }
1664
-}
1665
-
1666
-} // namespace marlin_moe
1667
-
1668
-torch::Tensor marlin_gemm_moe(
1669
- const torch::Tensor& a, const torch::Tensor& b_q_weights,
1670
- const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
1671
- const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
1672
- const torch::Tensor& g_idx, const torch::Tensor& perm,
1673
- torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
1674
- bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
1675
- bool replicate_input, bool apply_weights) {
1676
- int max_par = 4;
1677
-
1678
- int dev = a.get_device();
1679
-
1680
- auto options_dtype =
1681
- torch::TensorOptions().dtype(a.dtype()).device(a.device());
1682
- auto options_int =
1683
- torch::TensorOptions().dtype(torch::kInt).device(a.device());
1684
- torch::Tensor c = torch::zeros({size_m, topk, size_n}, options_dtype);
1685
- torch::Tensor a_tmp =
1686
- replicate_input ? torch::zeros({size_m, size_k}, options_dtype)
1687
- : torch::zeros({size_m, topk, size_k}, options_dtype);
1688
- torch::Tensor expert_offsets = torch::empty({num_experts + 1}, options_int);
1689
-
1690
- // thread_k: `k` size of a thread_tile in `weights` (can usually be left as
1691
- // auto -1)
1692
- int thread_k = -1;
1693
- // thread_n: `n` size of a thread_tile in `weights` (can usually be left as
1694
- // auto -1)
1695
- int thread_n = -1;
1696
- // sms: number of SMs to use for the kernel (can usually be left as auto -1)
1697
- int sms = -1;
1698
-
1699
- // Detect groupsize and act_order
1700
- int num_groups = -1;
1701
- int group_size = -1;
1702
- bool has_act_order = g_idx.size(1) != 0;
1703
-
1704
- int b_rank = b_scales.sizes().size();
1705
- TORCH_CHECK(b_rank == 3, "b_scales rank = ", b_rank, " is not 3");
1706
- TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2),
1707
- " is not size_n = ", size_n);
1708
- num_groups = b_scales.size(1);
1709
-
1710
- if (has_act_order) {
1711
- if (is_k_full) {
1712
- TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
1713
- TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k,
1714
- ", is not divisible by num_groups = ", num_groups);
1715
- group_size = size_k / num_groups;
1716
- } else {
1717
- group_size = 0;
1718
- }
1719
-
1720
- } else {
1721
- if (num_groups > 1) {
1722
- TORCH_CHECK(
1723
- size_k % num_groups == 0, "size_k = ", size_k,
1724
- ", is not divisible by b_scales.size(0) = ", b_scales.size(0));
1725
- group_size = size_k / num_groups;
1726
- } else {
1727
- group_size = -1;
1728
- }
1729
- }
1730
-
1731
- marlin_moe::marlin_mm_moe_f16i4(
1732
- a.data_ptr(), b_q_weights.data_ptr(), c.data_ptr(), sorted_ids.data_ptr(),
1733
- topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
1734
- g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
1735
- expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
1736
- has_act_order, is_k_full, num_groups, group_size, num_experts, topk,
1737
- moe_block_size, dev, at::cuda::getCurrentCUDAStream(dev), thread_k,
1738
- thread_n, sms, max_par, replicate_input, apply_weights);
1739
- return c;
1740
-}
\ No newline at end of file
csrc/moe/marlin_moe_ops.h
deleted
-12
@@ -1,12 +0,0 @@
1
-#pragma once
2
-
3
-#include <torch/all.h>
4
-
5
-torch::Tensor marlin_gemm_moe(
6
- const torch::Tensor& a, const torch::Tensor& b_q_weights,
7
- const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
8
- const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
9
- const torch::Tensor& g_idx, const torch::Tensor& perm,
10
- torch::Tensor& workspace, int64_t size_m, int64_t size_n, int64_t size_k,
11
- bool is_k_full, int64_t num_experts, int64_t topk, int64_t moe_block_size,
12
- bool replicate_input, bool apply_weights);
\ No newline at end of file
csrc/moe/torch_bindings.cpp
-9
@@ -1,6 +1,5 @@
1
#include "core/registration.h"
2
#include "moe_ops.h"
3
-#include "marlin_moe_ops.h"
3
4
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5
// Apply topk softmax to the gating outputs.
@@ -8,14 +7,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
7
"topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
8
"token_expert_indices, Tensor gating_output) -> ()");
9
m.impl("topk_softmax", torch::kCUDA, &topk_softmax);
11
- m.def(
12
- "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
13
- "Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
14
- "g_idx, Tensor! perm, Tensor! workspace, int size_m, int size_n, int "
15
- "size_k, bool is_k_full, int num_experts, int topk, int moe_block_size, "
16
- "bool replicate_input, bool apply_weights) -> Tensor");
17
-
18
- m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
10
}
11
12
REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
tests/weight_loading/models.txt
-2
@@ -13,7 +13,5 @@ compressed-tensors, nm-testing/tinyllama-oneshot-w8a16-per-channel, main
13
compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
14
compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
15
compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
16
-compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
17
-compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
16
awq, casperhansen/mixtral-instruct-awq, main
17
awq_marlin, casperhansen/mixtral-instruct-awq, main
vllm/_custom_ops.py
-14
@@ -300,20 +300,6 @@ def awq_marlin_repack(b_q_weight: torch.Tensor, size_k: int, size_n: int,
300
return torch.ops._C.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits)
301
302
303
-def gptq_marlin_moe_repack(b_q_weight: torch.Tensor, perm: torch.Tensor,
304
- size_k: int, size_n: int,
305
- num_bits: int) -> torch.Tensor:
306
- num_experts = b_q_weight.shape[0]
307
- assert size_k % 16 == 0
308
- output = torch.empty((num_experts, size_k // 16, size_n * 2),
309
- device=b_q_weight.device,
310
- dtype=b_q_weight.dtype)
311
- for e in range(num_experts):
312
- output[e] = torch.ops._C.gptq_marlin_repack(b_q_weight[e], perm[e],
313
- size_k, size_n, num_bits)
314
- return output
315
-
316
-
303
def gptq_marlin_gemm(a: torch.Tensor,
304
b_q_weight: torch.Tensor,
305
b_scales: torch.Tensor,
vllm/model_executor/layers/fused_moe/__init__.py
+8
-6
@@ -1,17 +1,19 @@
1
-from vllm.model_executor.layers.fused_moe.layer import (
2
- FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
1
+from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
2
+ FusedMoEMethodBase)
3
from vllm.triton_utils import HAS_TRITON
4
5
-__all__ = ["FusedMoE", "FusedMoEMethodBase", "FusedMoeWeightScaleSupported"]
5
+__all__ = [
6
+ "FusedMoE",
7
+ "FusedMoEMethodBase",
8
+]
9
10
if HAS_TRITON:
11
12
from vllm.model_executor.layers.fused_moe.fused_moe import (
10
- fused_experts, fused_marlin_moe, fused_moe, fused_topk,
11
- get_config_file_name, grouped_topk)
13
+ fused_experts, fused_moe, fused_topk, get_config_file_name,
14
+ grouped_topk)
15
16
__all__ += [
14
- "fused_marlin_moe",
17
"fused_moe",
18
"fused_topk",
19
"fused_experts",
vllm/model_executor/layers/fused_moe/fused_moe.py
+18
-116
@@ -323,16 +323,21 @@ def get_moe_configs(E: int, N: int,
323
return None
324
325
326
-def get_default_config(M: int, E: int, N: int, K: int, topk: int,
327
- dtype: Optional[str],
328
- is_marlin: bool) -> Dict[str, int]:
326
+def get_default_config(
327
+ M: int,
328
+ E: int,
329
+ N: int,
330
+ K: int,
331
+ topk: int,
332
+ dtype: Optional[str],
333
+) -> Dict[str, int]:
334
config = {
335
'BLOCK_SIZE_M': 64,
336
'BLOCK_SIZE_N': 64,
337
'BLOCK_SIZE_K': 32,
338
'GROUP_SIZE_M': 8
339
}
335
- if M <= E or (is_marlin and M <= 32):
340
+ if M <= E:
341
config = {
342
'BLOCK_SIZE_M': 16,
343
'BLOCK_SIZE_N': 32,
@@ -342,14 +347,14 @@ def get_default_config(M: int, E: int, N: int, K: int, topk: int,
347
return config
348
349
345
-def try_get_optimal_moe_config(w1_shape: Tuple[int, ...],
346
- w2_shape: Tuple[int, ...],
347
- top_k: int,
348
- dtype: Optional[str],
349
- M: int,
350
- override_config: Optional[Dict[str,
351
- Any]] = None,
352
- is_marlin: bool = False):
350
+def try_get_optimal_moe_config(
351
+ w1_shape: Tuple[int, ...],
352
+ w2_shape: Tuple[int, ...],
353
+ top_k: int,
354
+ dtype: Optional[str],
355
+ M: int,
356
+ override_config: Optional[Dict[str, Any]] = None,
357
+):
358
if override_config:
359
config = override_config
360
else:
@@ -363,8 +368,7 @@ def try_get_optimal_moe_config(w1_shape: Tuple[int, ...],
368
config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
369
else:
370
# Else use the default config
366
- config = get_default_config(M, E, N, w1_shape[2], top_k, dtype,
367
- is_marlin)
371
+ config = get_default_config(M, E, N, w1_shape[2], top_k, dtype)
372
return config
373
374
@@ -437,108 +441,6 @@ def grouped_topk(hidden_states: torch.Tensor,
441
return topk_weights, topk_ids
442
443
440
-def fused_marlin_moe(hidden_states: torch.Tensor,
441
- w1: torch.Tensor,
442
- w2: torch.Tensor,
443
- gating_output: torch.Tensor,
444
- g_idx1: torch.Tensor,
445
- g_idx2: torch.Tensor,
446
- rand_perm1: torch.Tensor,
447
- rand_perm2: torch.Tensor,
448
- topk: int,
449
- renormalize: bool,
450
- override_config: Optional[Dict[str, Any]] = None,
451
- use_fp8: bool = False,
452
- w1_scale: Optional[torch.Tensor] = None,
453
- w2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
454
- """
455
- This function computes a Mixture of Experts (MoE) layer using two sets of
456
- weights, w1 and w2, and top-k gating mechanism.
457
- Parameters:
458
- - hidden_states (torch.Tensor): The input tensor to the MoE layer.
459
- - w1 (torch.Tensor): The first set of expert weights.
460
- - w2 (torch.Tensor): The second set of expert weights.
461
- - gating_output (torch.Tensor): The output of the gating operation
462
- (before softmax).
463
- - topk (int): The number of top-k experts to select.
464
- - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
465
- - inplace (bool): If True, perform the operation in-place.
466
- Defaults to False.
467
- - override_config (Optional[Dict[str, Any]]): Optional override
468
- for the kernel configuration.
469
- - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
470
- products for w1 and w2. Defaults to False.
471
- - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
472
- w1.
473
- - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
474
- w2.
475
- Returns:
476
- - torch.Tensor: The output tensor after applying the MoE layer.
477
- """
478
- # Check constraints.
479
- assert hidden_states.shape[0] == gating_output.shape[0], (
480
- "Number of tokens mismatch")
481
- assert hidden_states.shape[
482
- 1] == w1.shape[1] * 16, "Hidden size mismatch w1"
483
- assert hidden_states.shape[
484
- 1] == w2.shape[2] // 2, "Hidden size mismatch w2"
485
- assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
486
- assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
487
- assert w1.is_contiguous(), "Expert weights1 must be contiguous"
488
- assert w2.is_contiguous(), "Expert weights2 must be contiguous"
489
- assert hidden_states.dtype in [
490
- torch.float32, torch.float16, torch.bfloat16
491
- ]
492
-
493
- #TODO fp8 is not implemented yet
494
- assert not use_fp8
495
-
496
- M, K = hidden_states.shape
497
- E = w1.shape[0]
498
- N = w2.shape[1] * 16
499
-
500
- topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
501
- renormalize)
502
-
503
- get_config_func = functools.partial(try_get_optimal_moe_config,
504
- w1.shape,
505
- w2.shape,
506
- topk_ids.shape[1],
507
- "float8" if use_fp8 else None,
508
- override_config=override_config,
509
- is_marlin=True)
510
- config = get_config_func(M)
511
-
512
- block_size_m = config['BLOCK_SIZE_M']
513
-
514
- sorted_token_ids, _, _ = moe_align_block_size(topk_ids, block_size_m, E)
515
-
516
- max_workspace_size = ((M + 255) // 256) * (max(2 * N, K) // 64) * 16
517
- workspace = torch.zeros(max_workspace_size,
518
- dtype=torch.int,
519
- device="cuda",
520
- requires_grad=False)
521
-
522
- intermediate_cache2 = torch.empty((M * topk_ids.shape[1], N),
523
- device=hidden_states.device,
524
- dtype=hidden_states.dtype)
525
-
526
- intermediate_cache1 = torch.ops._moe_C.marlin_gemm_moe(
527
- hidden_states, w1, sorted_token_ids, topk_weights, topk_ids, w1_scale,
528
- g_idx1, rand_perm1, workspace, M, 2 * N, K, True, E, topk,
529
- block_size_m, True, False)
530
-
531
- ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
532
-
533
- intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
534
- intermediate_cache2, w2, sorted_token_ids, topk_weights, topk_ids,
535
- w2_scale, g_idx2, rand_perm2, workspace, M, K, N, True, E, topk,
536
- block_size_m, False, True)
537
-
538
- return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
539
- dim=1)
540
-
541
-
444
def get_config_dtype_str(dtype: torch.dtype,
445
use_int8_w8a16: Optional[bool] = False,
446
use_fp8_w8a8: Optional[bool] = False):
vllm/model_executor/layers/fused_moe/layer.py
+36
-170
@@ -1,5 +1,4 @@
1
from abc import abstractmethod
2
-from enum import Enum
2
from typing import List, Optional, Tuple
3
4
import torch
@@ -16,12 +15,6 @@ from vllm.model_executor.utils import set_weight_attrs
15
logger = init_logger(__name__)
16
17
19
-class FusedMoeWeightScaleSupported(Enum):
20
- TENSOR = "tensor"
21
- CHANNEL = "channel"
22
- GROUP = "group"
23
-
24
-
18
class FusedMoEMethodBase(QuantizeMethodBase):
19
20
@abstractmethod
@@ -206,182 +199,55 @@ class FusedMoE(torch.nn.Module):
199
params_dtype=params_dtype,
200
weight_loader=self.weight_loader)
201
209
- def _load_per_tensor_weight_scale(self, shard_id: str,
210
- param: torch.nn.Parameter,
211
- loaded_weight: torch.Tensor,
212
- expert_id: int):
213
- param_data = param.data
214
- # for per tensor weight quantization
215
- if shard_id in ("w1", "w3"):
216
- # We have to keep the weight scales of w1 and w3 because
217
- # we need to re-quantize w1/w3 weights after weight loading.
218
- idx = 0 if shard_id == "w1" else 1
219
- param_data[expert_id][idx] = loaded_weight
220
- # If we are in the row parallel case (down_proj)
221
- elif shard_id == "w2":
222
- param_data[expert_id] = loaded_weight
223
-
224
- def _load_model_weight_or_group_weight_scale(self, shard_dim: int,
225
- expert_data: torch.Tensor,
226
- shard_id: str,
227
- loaded_weight: torch.tensor,
228
- tp_rank: int):
229
- # Load grouped weight scales for group quantization
230
- # or model weights
231
- if shard_id == "w2":
232
- self._load_w2(shard_id=shard_id,
233
- shard_dim=shard_dim,
234
- loaded_weight=loaded_weight,
235
- expert_data=expert_data,
236
- tp_rank=tp_rank)
237
- elif shard_id in ("w1", "w3"):
238
- self._load_w13(shard_id=shard_id,
239
- shard_dim=shard_dim,
240
- loaded_weight=loaded_weight,
241
- expert_data=expert_data,
242
- tp_rank=tp_rank)
243
-
244
- def _load_per_channel_weight_scale(self, expert_data: torch.Tensor,
245
- shard_dim: int, shard_id: str,
246
- loaded_weight: torch.tensor,
247
- tp_rank: int):
248
- # for per channel weight quantization
249
- if shard_id == "w2":
250
- expert_data.copy_(loaded_weight)
251
- elif shard_id in ("w1", "w3"):
252
- self._load_w13(shard_id=shard_id,
253
- shard_dim=shard_dim,
254
- loaded_weight=loaded_weight,
255
- expert_data=expert_data,
256
- tp_rank=tp_rank)
257
-
258
- def _load_w13(self, expert_data: torch.Tensor, shard_dim: int,
259
- shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
260
-
261
- # Index the loaded weight for tp sharding.
262
- # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
263
- shard_size = expert_data.shape[shard_dim] // 2
264
- loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
265
- shard_size)
266
- # Narrow parameter and load.
267
- # w1, gate_proj: Load into first logical weight of w13.
268
- if shard_id == "w1":
269
- expert_data = expert_data.narrow(shard_dim, 0, shard_size)
270
- # w3, up_proj: Load into second logical weight of w13.
271
- else:
272
- assert shard_id == "w3"
273
- expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
274
- expert_data.copy_(loaded_weight)
275
-
276
- def _load_w2(self, expert_data: torch.Tensor, shard_dim: int,
277
- shard_id: str, loaded_weight: torch.tensor, tp_rank: int):
278
-
279
- # Index the loaded weight for tp sharding.
280
- # down_proj: "RowParallel" so tp sharding on input_dim
281
- # Narrow parameter and load.
282
- shard_size = expert_data.shape[shard_dim]
283
- loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
284
- shard_size)
285
- # w2, down_proj: Load into only logical weight of w2.
286
- expert_data.copy_(loaded_weight)
287
-
288
- def _load_single_value(self, param: torch.nn.Parameter,
289
- loaded_weight: torch.Tensor, expert_id: int):
290
- param_data = param.data
291
-
292
- # Input scales can be loaded directly and should be equal.
293
- param_data[expert_id] = loaded_weight
294
-
202
def weight_loader(self, param: torch.nn.Parameter,
203
loaded_weight: torch.Tensor, weight_name: str,
204
shard_id: str, expert_id: int) -> None:
298
-
205
if shard_id not in ("w1", "w2", "w3"):
206
raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
207
f"got {shard_id}.")
208
303
- WEIGHT_SCALE_SUPPORTED = [
304
- e.value for e in FusedMoeWeightScaleSupported
305
- ]
306
- # Fetch the dim to shard the parameter/loaded weight
307
- # based on the shard id. This will be whatever
308
- # dimension intermediate_size is used.
309
- SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
209
+ # Special case for fp8 scales.
210
+ if getattr(param, "is_fp8_scale", False):
211
+ self._load_fp8_scale(param.data, loaded_weight, weight_name,
212
+ shard_id, expert_id)
213
+ return
214
215
expert_data = param.data[expert_id]
216
tp_rank = get_tensor_model_parallel_rank()
217
314
- # is_transposed: whether or not the parameter is transposed on disk
315
- # If transposed, the loaded weight will be transposed and the dim
316
- # to shard the loaded weight will be flipped.
317
- is_transposed = getattr(param, "is_transposed", False)
318
- shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
319
- if is_transposed:
320
- loaded_weight = loaded_weight.t().contiguous()
321
- shard_dim = ~shard_dim
322
-
323
- # Case weight_scales
324
- if "weight_scale" in weight_name:
325
- # load the weight scaling based on the quantization scheme
326
- # supported weight scales can be found in
327
- # FusedMoeWeightScaleSupported
328
- # TODO @dsikka: once hardened, refactor to use vLLM Parameters
329
- # specific to each case
330
- quant_method = getattr(param, "quant_method", None)
331
- if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value:
332
- self._load_per_channel_weight_scale(
333
- shard_id=shard_id,
334
- shard_dim=shard_dim,
335
- loaded_weight=loaded_weight,
336
- expert_data=expert_data,
337
- tp_rank=tp_rank)
338
- elif quant_method == FusedMoeWeightScaleSupported.GROUP.value:
339
- self._load_model_weight_or_group_weight_scale(
340
- shard_id=shard_id,
341
- shard_dim=shard_dim,
342
- loaded_weight=loaded_weight,
343
- expert_data=expert_data,
344
- tp_rank=tp_rank)
345
- elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
346
- self._load_per_tensor_weight_scale(shard_id=shard_id,
347
- param=param,
348
- loaded_weight=loaded_weight,
349
- expert_id=expert_id)
350
- else:
351
- raise ValueError(
352
- f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}")
353
- return
354
-
355
- if "weight_shape" in weight_name:
356
- self._load_single_value(param=param,
357
- loaded_weight=loaded_weight,
358
- expert_id=expert_id)
359
- return
218
+ # If transposed, weight is saved as [input_dim, output_dim]
219
+ # Otherwise, weight is saved as [output_dim, input_dim]
220
+ # Default is not transposed/input dim is dim 1
221
+ input_dim = getattr(param, "input_dim", 1)
222
+ output_dim = getattr(param, "output_dim", 0)
223
361
- # Case input scale
362
- if "input_scale" in weight_name:
363
- # Note: input_scale loading is only supported for fp8
364
- if param.data[expert_id] != 1 and (param.data[expert_id] -
365
- loaded_weight).abs() > 1e-5:
366
- raise ValueError(
367
- "input_scales of w1 and w3 of a layer "
368
- f"must be equal. But got {param.data[expert_id]} "
369
- f"vs. {loaded_weight}")
370
-
371
- self._load_single_value(param=param,
372
- loaded_weight=loaded_weight,
373
- expert_id=expert_id)
374
- return
224
+ # Index the loaded weight for tp sharding.
225
+ # down_proj: "RowParallel" so tp sharding on input_dim
226
+ if shard_id == "w2":
227
+ shard_dim = input_dim
228
+ shard_size = expert_data.shape[shard_dim]
229
+ # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
230
+ elif shard_id in ("w1", "w3"):
231
+ shard_dim = output_dim
232
+ shard_size = expert_data.shape[output_dim] // 2
233
+ offset = shard_size * tp_rank
234
+ loaded_weight = loaded_weight.narrow(shard_dim, offset, shard_size)
235
376
- # Case model weights
377
- if "weight" in weight_name:
378
- self._load_model_weight_or_group_weight_scale(
379
- shard_id=shard_id,
380
- shard_dim=shard_dim,
381
- loaded_weight=loaded_weight,
382
- expert_data=expert_data,
383
- tp_rank=tp_rank)
384
- return
236
+ # Narrow parameter and load.
237
+ # w1, gate_proj: Load into first logical weight of w13.
238
+ if shard_id == "w1":
239
+ expert_data = expert_data.narrow(shard_dim, 0, shard_size)
240
+ expert_data.copy_(loaded_weight)
241
+ # w3, up_proj: Load into second logical weight of w13.
242
+ elif shard_id == "w3":
243
+ expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
244
+ expert_data.copy_(loaded_weight)
245
+ # w2, down_proj: Load into only logical weight of w2.
246
+ elif shard_id == "w2":
247
+ expert_data.copy_(loaded_weight)
248
+ else:
249
+ raise ValueError(
250
+ f"Expected shard_id w1,w2 or w3 but got {shard_id}")
251
252
@staticmethod
253
def select_experts(hidden_states: torch.Tensor,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
-5
@@ -3,12 +3,9 @@ from typing import Any, Dict, List, Optional
3
import torch
4
from pydantic import BaseModel
5
6
-from vllm.model_executor.layers.fused_moe import FusedMoE
6
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
7
from vllm.model_executor.layers.quantization.base_config import ( # noqa: E501
8
QuantizationConfig, QuantizeMethodBase)
10
-from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501
11
- CompressedTensorsMoEMethod)
9
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
W4A16SPARSE24_SUPPORTED_BITS, WNA16_SUPPORTED_BITS,
11
CompressedTensorsScheme, CompressedTensorsUnquantized,
@@ -67,8 +64,6 @@ class CompressedTensorsConfig(QuantizationConfig):
64
return CompressedTensorsLinearMethod(self)
65
if isinstance(layer, Attention):
66
return CompressedTensorsKVCacheMethod(self)
70
- if isinstance(layer, FusedMoE):
71
- return CompressedTensorsMoEMethod(self)
67
return None
68
69
@classmethod
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
deleted
-283
@@ -1,283 +0,0 @@
1
-import enum
2
-from enum import Enum
3
-from typing import List, Optional
4
-
5
-import torch
6
-
7
-from vllm import _custom_ops as ops
8
-from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
9
-from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
10
- WNA16_SUPPORTED_BITS)
11
-from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
12
- CompressionFormat)
13
-from vllm.model_executor.utils import set_weight_attrs
14
-
15
-
16
-class GPTQMarlinState(Enum):
17
- REPACK = enum.auto()
18
- READY = enum.auto()
19
-
20
-
21
-__all__ = ["CompressedTensorsMoEMethod"]
22
-
23
-
24
-class CompressedTensorsMoEMethod(FusedMoEMethodBase):
25
-
26
- def __init__(
27
- self,
28
- quant_config: "CompressedTensorsConfig" # type: ignore # noqa E501
29
- ):
30
- self.quant_config = quant_config
31
- # TODO: @dsikka: refactor this to use schemes as other kernels
32
- # are supported + check if the layer is being ignored.
33
- config = self.quant_config.target_scheme_map["Linear"].get("weights")
34
- self.num_bits = config.num_bits
35
- self.packed_factor = 32 // config.num_bits
36
- self.strategy = config.strategy.value
37
- self.group_size = config.group_size
38
- assert config.symmetric, (
39
- "Only symmetric quantization is supported for MoE")
40
-
41
- if not (self.quant_config.quant_format
42
- == CompressionFormat.pack_quantized.value
43
- and self.num_bits in WNA16_SUPPORTED_BITS):
44
- raise ValueError("For Fused MoE layers, only ",
45
- f"{CompressionFormat.pack_quantized.value} ",
46
- "is supported for the following bits: ",
47
- f"{WNA16_SUPPORTED_BITS}")
48
-
49
- def create_weights(self, layer: torch.nn.Module, num_experts: int,
50
- hidden_size: int, intermediate_size: int,
51
- params_dtype: torch.dtype, **extra_weight_attrs):
52
-
53
- # Will transpose the loaded weight along the
54
- # intermediate and hidden dim sizes. Will
55
- # shard for TP along the transposed dims
56
- extra_weight_attrs.update({
57
- "is_transposed": True,
58
- "quant_method": self.strategy
59
- })
60
- w13_weight = torch.nn.Parameter(torch.empty(num_experts,
61
- hidden_size //
62
- self.packed_factor,
63
- 2 * intermediate_size,
64
- dtype=torch.int32),
65
- requires_grad=False)
66
- layer.register_parameter("w13_weight_packed", w13_weight)
67
- set_weight_attrs(w13_weight, extra_weight_attrs)
68
-
69
- w2_weight = torch.nn.Parameter(torch.empty(num_experts,
70
- intermediate_size //
71
- self.packed_factor,
72
- hidden_size,
73
- dtype=torch.int32),
74
- requires_grad=False)
75
- layer.register_parameter("w2_weight_packed", w2_weight)
76
- set_weight_attrs(w2_weight, extra_weight_attrs)
77
-
78
- if self.strategy == "channel":
79
- num_groups_w2 = num_groups_w13 = 1
80
- self.group_size = -1
81
- else:
82
- num_groups_w2 = intermediate_size // self.group_size
83
- num_groups_w13 = hidden_size // self.group_size
84
-
85
- w13_scale = torch.nn.Parameter(torch.ones(num_experts,
86
- num_groups_w13,
87
- 2 * intermediate_size,
88
- dtype=params_dtype),
89
- requires_grad=False)
90
- layer.register_parameter("w13_weight_scale", w13_scale)
91
- set_weight_attrs(w13_scale, extra_weight_attrs)
92
-
93
- w2_scale = torch.nn.Parameter(torch.ones(num_experts,
94
- num_groups_w2,
95
- hidden_size,
96
- dtype=params_dtype),
97
- requires_grad=False)
98
- layer.register_parameter("w2_weight_scale", w2_scale)
99
- set_weight_attrs(w2_scale, extra_weight_attrs)
100
-
101
- w2_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
102
- requires_grad=False)
103
- layer.register_parameter("w2_weight_shape", w2_weight_shape)
104
- set_weight_attrs(w2_weight_shape, extra_weight_attrs)
105
- w13_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
106
- requires_grad=False)
107
-
108
- layer.register_parameter("w13_weight_shape", w13_weight_shape)
109
- set_weight_attrs(w13_weight_shape, extra_weight_attrs)
110
-
111
- w13_g_idx = torch.nn.Parameter(
112
- torch.empty(
113
- num_experts,
114
- hidden_size,
115
- dtype=torch.int32,
116
- ),
117
- requires_grad=False,
118
- )
119
- layer.register_parameter("w13_g_idx", w13_g_idx)
120
- set_weight_attrs(w13_g_idx, extra_weight_attrs)
121
-
122
- w2_g_idx = torch.nn.Parameter(
123
- torch.empty(
124
- num_experts,
125
- intermediate_size,
126
- dtype=torch.int32,
127
- ),
128
- requires_grad=False,
129
- )
130
- layer.register_parameter("w2_g_idx", w2_g_idx)
131
- set_weight_attrs(w2_g_idx, extra_weight_attrs)
132
-
133
- w13_g_idx_sort_indices = torch.nn.Parameter(
134
- torch.empty(
135
- num_experts,
136
- hidden_size,
137
- dtype=torch.int32,
138
- ),
139
- requires_grad=False,
140
- )
141
- layer.register_parameter("w13_g_idx_sort_indices",
142
- w13_g_idx_sort_indices)
143
- set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs)
144
-
145
- w2_g_idx_sort_indices = torch.nn.Parameter(
146
- torch.empty(
147
- num_experts,
148
- intermediate_size,
149
- dtype=torch.int32,
150
- ),
151
- requires_grad=False,
152
- )
153
- layer.register_parameter("w2_g_idx_sort_indices",
154
- w2_g_idx_sort_indices)
155
- set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
156
-
157
- layer.a13_scale = None
158
- layer.a2_scale = None
159
- layer.marlin_state = GPTQMarlinState.REPACK
160
-
161
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
162
-
163
- def replace_tensor(name, new_t):
164
- # It is important to use resize_() here since it ensures
165
- # the same buffer is reused
166
- getattr(layer, name).resize_(new_t.shape)
167
- getattr(layer, name).copy_(new_t)
168
- del new_t
169
-
170
- def get_scale_perms(num_bits: int):
171
- scale_perm: List[int] = []
172
- for i in range(8):
173
- scale_perm.extend([i + 8 * j for j in range(8)])
174
- scale_perm_single: List[int] = []
175
- for i in range(4):
176
- scale_perm_single.extend(
177
- [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
178
- return scale_perm, scale_perm_single
179
-
180
- def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
181
- group_size: int, num_bits: int):
182
- scale_perm, scale_perm_single = get_scale_perms(num_bits)
183
- if group_size < size_k and group_size != -1:
184
- s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
185
- else:
186
- s = s.reshape((-1, len(scale_perm_single)))[:,
187
- scale_perm_single]
188
- s = s.reshape((-1, size_n)).contiguous()
189
- return s
190
-
191
- def marlin_moe_permute_scales(s: torch.Tensor, size_k: int,
192
- size_n: int, group_size: int,
193
- num_bits: int):
194
- num_experts = s.shape[0]
195
- output = torch.empty((num_experts, s.shape[1], s.shape[2]),
196
- device=s.device,
197
- dtype=s.dtype)
198
- for e in range(num_experts):
199
- output[e] = marlin_permute_scales(s[e], size_k, size_n,
200
- group_size, num_bits)
201
- return output
202
-
203
- size_k2 = layer.w2_weight_packed.shape[2]
204
- size_k13 = layer.w13_weight_packed.shape[2]
205
-
206
- num_experts = layer.w13_g_idx.shape[0]
207
- device = layer.w13_g_idx.device
208
- layer.w13_g_idx = torch.nn.Parameter(
209
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
210
- requires_grad=False,
211
- )
212
- layer.w2_g_idx = torch.nn.Parameter(
213
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
214
- requires_grad=False,
215
- )
216
- layer.w13_g_idx_sort_indices = torch.nn.Parameter(
217
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
218
- requires_grad=False,
219
- )
220
- layer.w2_g_idx_sort_indices = torch.nn.Parameter(
221
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
222
- requires_grad=False,
223
- )
224
-
225
- marlin_w13_qweight = ops.gptq_marlin_moe_repack(
226
- layer.w13_weight_packed,
227
- layer.w13_g_idx_sort_indices,
228
- layer.w13_weight_packed.shape[1] * self.packed_factor,
229
- layer.w13_weight_packed.shape[2],
230
- self.num_bits,
231
- )
232
- replace_tensor("w13_weight_packed", marlin_w13_qweight)
233
- marlin_w2_qweight = ops.gptq_marlin_moe_repack(
234
- layer.w2_weight_packed,
235
- layer.w2_g_idx_sort_indices,
236
- layer.w2_weight_packed.shape[1] * self.packed_factor,
237
- layer.w2_weight_packed.shape[2],
238
- self.num_bits,
239
- )
240
- replace_tensor("w2_weight_packed", marlin_w2_qweight)
241
- # Repack scales
242
- marlin_w13_scales = marlin_moe_permute_scales(
243
- layer.w13_weight_scale,
244
- size_k13,
245
- layer.w13_weight_scale.shape[2],
246
- self.group_size,
247
- self.num_bits,
248
- )
249
- replace_tensor("w13_weight_scale", marlin_w13_scales)
250
- marlin_w2_scales = marlin_moe_permute_scales(
251
- layer.w2_weight_scale,
252
- layer.w2_weight_scale.shape[1] * self.packed_factor,
253
- size_k2,
254
- self.group_size,
255
- self.num_bits,
256
- )
257
- replace_tensor("w2_weight_scale", marlin_w2_scales)
258
-
259
- def apply(self,
260
- layer: torch.nn.Module,
261
- x: torch.Tensor,
262
- router_logits: torch.Tensor,
263
- top_k: int,
264
- renormalize: bool = True,
265
- use_grouped_topk: bool = False,
266
- num_expert_group: Optional[int] = None,
267
- topk_group: Optional[int] = None) -> torch.Tensor:
268
-
269
- from vllm.model_executor.layers.fused_moe.fused_moe import (
270
- fused_marlin_moe)
271
-
272
- return fused_marlin_moe(x,
273
- layer.w13_weight_packed,
274
- layer.w2_weight_packed,
275
- router_logits,
276
- layer.w13_g_idx,
277
- layer.w2_g_idx,
278
- layer.w13_g_idx_sort_indices,
279
- layer.w2_g_idx_sort_indices,
280
- top_k,
281
- renormalize=renormalize,
282
- w1_scale=layer.w13_weight_scale,
283
- w2_scale=layer.w2_weight_scale)
vllm/model_executor/layers/quantization/fp8.py
+18
-11
@@ -7,8 +7,7 @@ from torch.nn.parameter import Parameter
7
import vllm.envs as envs
8
from vllm import _custom_ops as ops
9
from vllm.logger import init_logger
10
-from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11
- FusedMoeWeightScaleSupported)
10
+from vllm.model_executor.layers.fused_moe import FusedMoE, FusedMoEMethodBase
11
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
12
UnquantizedLinearMethod)
13
from vllm.model_executor.layers.quantization.base_config import (
@@ -319,16 +318,19 @@ class Fp8MoEMethod(FusedMoEMethodBase):
318
dtype=torch.float32),
319
requires_grad=False)
320
layer.register_parameter("w2_weight_scale", w2_weight_scale)
322
- # Add the quantization method used (per tensor/grouped/channel)
323
- # to ensure the weight scales are loaded in properly
324
- extra_weight_attrs.update(
325
- {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value})
321
+
322
# If loading fp8 checkpoint, pass the weight loaders.
323
# If loading an fp16 checkpoint, do not (we will quantize in
324
# process_weights_after_loading()
325
if self.quant_config.is_checkpoint_fp8_serialized:
330
- set_weight_attrs(w13_weight_scale, extra_weight_attrs)
331
- set_weight_attrs(w2_weight_scale, extra_weight_attrs)
326
+ set_weight_attrs(w13_weight_scale, {
327
+ "is_fp8_scale": True,
328
+ **extra_weight_attrs
329
+ })
330
+ set_weight_attrs(w2_weight_scale, {
331
+ "is_fp8_scale": True,
332
+ **extra_weight_attrs
333
+ })
334
335
# INPUT_SCALES
336
if self.quant_config.activation_scheme == "static":
@@ -341,14 +343,19 @@ class Fp8MoEMethod(FusedMoEMethodBase):
343
num_experts, dtype=torch.float32),
344
requires_grad=False)
345
layer.register_parameter("w13_input_scale", w13_input_scale)
344
- set_weight_attrs(w13_input_scale, extra_weight_attrs)
346
+ set_weight_attrs(w13_input_scale, {
347
+ "is_fp8_scale": True,
348
+ **extra_weight_attrs
349
+ })
350
351
w2_input_scale = torch.nn.Parameter(torch.ones(
352
num_experts, dtype=torch.float32),
353
requires_grad=False)
354
layer.register_parameter("w2_input_scale", w2_input_scale)
350
- set_weight_attrs(w2_input_scale, extra_weight_attrs)
351
-
355
+ set_weight_attrs(w2_input_scale, {
356
+ "is_fp8_scale": True,
357
+ **extra_weight_attrs
358
+ })
359
else:
360
layer.w13_input_scale = None
361
layer.w2_input_scale = None
vllm/model_executor/model_loader/utils.py
+2
-2
@@ -23,11 +23,11 @@ def get_model_architecture(
23
architectures = getattr(model_config.hf_config, "architectures", [])
24
# Special handling for quantized Mixtral.
25
# FIXME(woosuk): This is a temporary hack.
26
- mixtral_supported = ["fp8", "compressed-tensors"]
26
if (model_config.quantization is not None
28
- and model_config.quantization not in mixtral_supported
27
+ and model_config.quantization != "fp8"
28
and "MixtralForCausalLM" in architectures):
29
architectures = ["QuantMixtralForCausalLM"]
30
+
31
return ModelRegistry.resolve_model_cls(architectures)
32
33
vllm/model_executor/models/jamba.py
+1
-1
@@ -920,7 +920,7 @@ class JambaForCausalLM(nn.Module, HasInnerState):
920
weight_loader = param.weight_loader
921
weight_loader(param,
922
loaded_weight,
923
- name,
923
+ weight_name,
924
shard_id=shard_id,
925
expert_id=expert_id)
926
break
vllm/model_executor/models/mixtral.py
-1
@@ -73,7 +73,6 @@ class MixtralMoE(nn.Module):
73
self.hidden_size = hidden_size
74
75
# Gate always runs at half / full precision for now.
76
-
76
self.gate = ReplicatedLinear(hidden_size,
77
num_experts,
78
bias=False,