A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Kernel] Expand FP8 support to Ampere GPUs using FP8 Marlin (#5975)
Michael Goin committed
Jul 3, 2024 at 13:38 UTC
47f0954af0a5aefd0db19875f6bdcbe933d055a9
11 files changed
+1587
-44
CMakeLists.txt
+1
@@ -171,6 +171,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
171
"csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
172
"csrc/quantization/gptq_marlin/gptq_marlin.cu"
173
"csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
174
+ "csrc/quantization/fp8/fp8_marlin.cu"
175
"csrc/custom_all_reduce.cu"
176
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
177
"csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu"
csrc/ops.h
+5
@@ -93,6 +93,11 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
93
int64_t size_k, int64_t size_n,
94
int64_t num_bits);
95
96
+torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
97
+ torch::Tensor& b_scales, torch::Tensor& workspace,
98
+ int64_t num_bits, int64_t size_m, int64_t size_n,
99
+ int64_t size_k);
100
+
101
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
102
103
void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
csrc/quantization/fp8/fp8_marlin.cu
new
+1308
@@ -0,0 +1,1308 @@
1
+/*
2
+ * Modified by Neural Magic
3
+ * Copyright (C) Marlin.2024 Elias Frantar
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * http://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+
18
+/*
19
+ * Adapted from https://github.com/IST-DASLab/marlin
20
+ */
21
+
22
+#include "../gptq_marlin/gptq_marlin.cuh"
23
+#include "../gptq_marlin/gptq_marlin_dtypes.cuh"
24
+
25
+using namespace gptq_marlin;
26
+
27
+#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
28
+ static_assert(std::is_same<scalar_t, half>::value || \
29
+ std::is_same<scalar_t, nv_bfloat16>::value, \
30
+ "only float16 and bfloat16 is supported");
31
+
32
+template <typename T>
33
+inline std::string str(T x) {
34
+ return std::to_string(x);
35
+}
36
+
37
+namespace fp8_marlin {
38
+
39
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
40
+
41
+template <typename scalar_t, // compute dtype, half or nv_float16
42
+ const int num_bits, // number of bits used for weights
43
+ const int threads, // number of threads in a threadblock
44
+ const int thread_m_blocks, // number of 16x16 blocks in the m
45
+ // dimension (batchsize) of the
46
+ // threadblock
47
+ const int thread_n_blocks, // same for n dimension (output)
48
+ const int thread_k_blocks, // same for k dimension (reduction)
49
+ const int stages, // number of stages for the async global->shared
50
+ // fetch pipeline
51
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
52
+ // with a separate quantization scale
53
+ >
54
+__global__ void Marlin(
55
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
56
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
57
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
58
+ const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
59
+ // (k/groupsize)xn
60
+ int num_groups, // number of scale groups per output channel
61
+ int prob_m, // batch dimension m
62
+ int prob_n, // output dimension n
63
+ int prob_k, // reduction dimension k
64
+ int* locks // extra global storage for barrier synchronization
65
+) {}
66
+
67
+} // namespace fp8_marlin
68
+
69
+torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
70
+ torch::Tensor& b_scales, torch::Tensor& workspace,
71
+ int64_t num_bits, int64_t size_m, int64_t size_n,
72
+ int64_t size_k) {
73
+ TORCH_CHECK_NOT_IMPLEMENTED(false,
74
+ "marlin_gemm(..) requires CUDA_ARCH >= 8.0");
75
+ return torch::empty({1, 1});
76
+}
77
+
78
+#else
79
+
80
+// m16n8k16 tensor core mma instruction with fp16 inputs and fp32
81
+// output/accumulation.
82
+template <typename scalar_t>
83
+__device__ inline void mma(const typename ScalarType<scalar_t>::FragA& a_frag,
84
+ const typename ScalarType<scalar_t>::FragB& frag_b,
85
+ typename ScalarType<scalar_t>::FragC& frag_c) {
86
+ const uint32_t* a = reinterpret_cast<const uint32_t*>(&a_frag);
87
+ const uint32_t* b = reinterpret_cast<const uint32_t*>(&frag_b);
88
+ float* c = reinterpret_cast<float*>(&frag_c);
89
+ if constexpr (std::is_same<scalar_t, half>::value) {
90
+ asm volatile(
91
+ "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 "
92
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
93
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
94
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
95
+ "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
96
+ } else if constexpr (std::is_same<scalar_t, nv_bfloat16>::value) {
97
+ asm volatile(
98
+ "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
99
+ "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
100
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
101
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
102
+ "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
103
+ } else {
104
+ STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
105
+ }
106
+}
107
+
108
+// Instruction for loading a full 16x16 matrix fragment of operand A from shared
109
+// memory, directly in tensor core layout.
110
+template <typename scalar_t>
111
+__device__ inline void ldsm4(typename ScalarType<scalar_t>::FragA& frag_a,
112
+ const void* smem_ptr) {
113
+ uint32_t* a = reinterpret_cast<uint32_t*>(&frag_a);
114
+ uint32_t smem = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
115
+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
116
+ : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
117
+ : "r"(smem));
118
+}
119
+
120
+// Fast FP8ToFp16/FP8ToBf16: Efficiently dequantize 8bit fp8_e4m3 values to fp16
121
+// bf16 Reference:
122
+// - FP16:
123
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85
124
+// - BF16:
125
+// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175
126
+template <typename scalar_t>
127
+__device__ inline typename ScalarType<scalar_t>::FragB dequant_8bit(int q) {
128
+ STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t);
129
+}
130
+
131
+template <>
132
+__device__ inline typename ScalarType<half>::FragB dequant_8bit<half>(int q) {
133
+ // Constants for FP8 (E4M3) and FP16 formats
134
+ constexpr int FP8_EXPONENT = 4, FP8_MANTISSA = 3, FP16_EXPONENT = 5;
135
+ constexpr int RIGHT_SHIFT = FP16_EXPONENT - FP8_EXPONENT;
136
+
137
+ // Calculate MASK for extracting mantissa and exponent
138
+ constexpr int MASK1 = 0x80000000;
139
+ constexpr int MASK2 = MASK1 >> (FP8_EXPONENT + FP8_MANTISSA);
140
+ constexpr int MASK3 = MASK2 & 0x7fffffff;
141
+ constexpr int MASK = MASK3 | (MASK3 >> 16);
142
+ // Final MASK value: 0x7F007F00
143
+
144
+ // Extract and shift FP8 values to FP16 format
145
+ int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT);
146
+ int Out2 = ((q << 8) & 0x80008000) | (((q << 8) & MASK) >> RIGHT_SHIFT);
147
+
148
+ // Construct and apply exponent bias
149
+ constexpr int BIAS_OFFSET =
150
+ (1 << (FP16_EXPONENT - 1)) - (1 << (FP8_EXPONENT - 1));
151
+ const half2 bias_reg = __float2half2_rn(float(1 << BIAS_OFFSET));
152
+
153
+ // Convert to half2 and apply bias
154
+ typename ScalarType<half>::FragB frag_b;
155
+ // Note: reverse indexing is intentional because weights are permuted
156
+ frag_b[1] = __hmul2(*reinterpret_cast<const half2*>(&Out1), bias_reg);
157
+ frag_b[0] = __hmul2(*reinterpret_cast<const half2*>(&Out2), bias_reg);
158
+ return frag_b;
159
+}
160
+
161
+template <>
162
+__device__ inline typename ScalarType<nv_bfloat16>::FragB
163
+dequant_8bit<nv_bfloat16>(int q) {
164
+ // Constants for FP8 (E4M3) and BF16 formats
165
+ constexpr int FP8_EXPONENT = 4, FP8_MANTISSA = 3, BF16_EXPONENT = 8;
166
+ constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP8_EXPONENT;
167
+
168
+ // Calculate MASK for extracting mantissa and exponent
169
+ constexpr int MASK1 = 0x80000000;
170
+ constexpr int MASK2 = MASK1 >> (FP8_EXPONENT + FP8_MANTISSA);
171
+ constexpr int MASK3 = MASK2 & 0x7fffffff;
172
+ constexpr int MASK = MASK3 | (MASK3 >> 16);
173
+ // Final MASK value: 0x7F007F00
174
+
175
+ // Extract and shift FP8 values to BF16 format
176
+ int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT);
177
+ int Out2 = ((q << 8) & 0x80008000) | (((q << 8) & MASK) >> RIGHT_SHIFT);
178
+
179
+ // Construct and apply exponent bias
180
+ constexpr int BIAS_OFFSET =
181
+ (1 << (BF16_EXPONENT - 1)) - (1 << (FP8_EXPONENT - 1));
182
+ // Add 127 (float exponent bias) to BIAS_OFFSET and shift to float exponent
183
+ // position
184
+ constexpr uint32_t BIAS = (BIAS_OFFSET + 127) << 23;
185
+ const nv_bfloat162 bias_reg =
186
+ __float2bfloat162_rn(*reinterpret_cast<const float*>(&BIAS));
187
+
188
+ // Convert to bfloat162 and apply bias
189
+ typename ScalarType<nv_bfloat16>::FragB frag_b;
190
+ // Note: reverse indexing is intentional because weights are permuted
191
+ frag_b[1] = __hmul2(*reinterpret_cast<const nv_bfloat162*>(&Out1), bias_reg);
192
+ frag_b[0] = __hmul2(*reinterpret_cast<const nv_bfloat162*>(&Out2), bias_reg);
193
+ return frag_b;
194
+}
195
+
196
+// Multiply dequantized values by the corresponding quantization scale; used
197
+// only for grouped quantization.
198
+template <typename scalar_t>
199
+__device__ inline void scale(typename ScalarType<scalar_t>::FragB& frag_b,
200
+ typename ScalarType<scalar_t>::FragS& frag_s,
201
+ int i) {
202
+ using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
203
+ scalar_t2 s =
204
+ ScalarType<scalar_t>::num2num2(reinterpret_cast<scalar_t*>(&frag_s)[i]);
205
+ frag_b[0] = __hmul2(frag_b[0], s);
206
+ frag_b[1] = __hmul2(frag_b[1], s);
207
+}
208
+
209
+// Given 2 floats multiply by 2 scales (halves)
210
+template <typename scalar_t>
211
+__device__ inline void scale_float(float* c,
212
+ typename ScalarType<scalar_t>::FragS& s) {
213
+ scalar_t* s_ptr = reinterpret_cast<scalar_t*>(&s);
214
+ c[0] = __fmul_rn(c[0], ScalarType<scalar_t>::num2float(s_ptr[0]));
215
+ c[1] = __fmul_rn(c[1], ScalarType<scalar_t>::num2float(s_ptr[1]));
216
+}
217
+
218
+// Wait until barrier reaches `count`, then lock for current threadblock.
219
+__device__ inline void barrier_acquire(int* lock, int count) {
220
+ if (threadIdx.x == 0) {
221
+ int state = -1;
222
+ do
223
+ // Guarantee that subsequent writes by this threadblock will be visible
224
+ // globally.
225
+ asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n"
226
+ : "=r"(state)
227
+ : "l"(lock));
228
+ while (state != count);
229
+ }
230
+ __syncthreads();
231
+}
232
+
233
+// Release barrier and increment visitation count.
234
+__device__ inline void barrier_release(int* lock, bool reset = false) {
235
+ __syncthreads();
236
+ if (threadIdx.x == 0) {
237
+ if (reset) {
238
+ lock[0] = 0;
239
+ return;
240
+ }
241
+ int val = 1;
242
+ // Make sure that all writes since acquiring this barrier are visible
243
+ // globally, while releasing the barrier.
244
+ asm volatile("fence.acq_rel.gpu;\n");
245
+ asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n"
246
+ :
247
+ : "l"(lock), "r"(val));
248
+ }
249
+}
250
+
251
+template <typename scalar_t, // compute dtype, half or nv_float16
252
+ const int num_bits, // number of bits used for weights
253
+ const int threads, // number of threads in a threadblock
254
+ const int thread_m_blocks, // number of 16x16 blocks in the m
255
+ // dimension (batchsize) of the
256
+ // threadblock
257
+ const int thread_n_blocks, // same for n dimension (output)
258
+ const int thread_k_blocks, // same for k dimension (reduction)
259
+ const int stages, // number of stages for the async global->shared
260
+ // fetch pipeline
261
+ const int group_blocks = -1 // number of consecutive 16x16 blocks
262
+ // with a separate quantization scale
263
+ >
264
+__global__ void Marlin(
265
+ const int4* __restrict__ A, // fp16 input matrix of shape mxk
266
+ const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn
267
+ int4* __restrict__ C, // fp16 output buffer of shape mxn
268
+ const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape
269
+ // (k/groupsize)xn
270
+ int num_groups, // number of scale groups per output channel
271
+ int prob_m, // batch dimension m
272
+ int prob_n, // output dimension n
273
+ int prob_k, // reduction dimension k
274
+ int* locks // extra global storage for barrier synchronization
275
+) {
276
+ // Each threadblock processes one "stripe" of the B matrix with (roughly) the
277
+ // same size, which might involve multiple column "slices" (of width 16 *
278
+ // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM
279
+ // example:
280
+ // 0 1 3
281
+ // 0 2 3
282
+ // 1 2 4
283
+ // While this kind of partitioning makes things somewhat more complicated, it
284
+ // ensures good utilization of all SMs for many kinds of shape and GPU
285
+ // configurations, while requiring as few slow global cross-threadblock
286
+ // reductions as possible.
287
+ using Dtype = ScalarType<scalar_t>;
288
+ using scalar_t2 = typename ScalarType<scalar_t>::scalar_t2;
289
+ using FragA = typename ScalarType<scalar_t>::FragA;
290
+ using FragB = typename ScalarType<scalar_t>::FragB;
291
+ using FragC = typename ScalarType<scalar_t>::FragC;
292
+ using FragS = typename ScalarType<scalar_t>::FragS;
293
+
294
+ constexpr int pack_factor = 32 / num_bits;
295
+
296
+ // For larger GEMMs we run multiple batchsize 64 versions in parallel for a
297
+ // better partitioning with less reductions
298
+ int parallel = 1;
299
+ if (prob_m > 16 * thread_m_blocks) {
300
+ parallel = prob_m / (16 * thread_m_blocks);
301
+ prob_m = 16 * thread_m_blocks;
302
+ }
303
+
304
+ int k_tiles = prob_k / 16 / thread_k_blocks;
305
+ int n_tiles = prob_n / 16 / thread_n_blocks;
306
+ int iters = div_ceil(k_tiles * n_tiles * parallel, gridDim.x);
307
+
308
+ int slice_row = (iters * blockIdx.x) % k_tiles;
309
+ int slice_col_par = (iters * blockIdx.x) / k_tiles;
310
+ int slice_col = slice_col_par;
311
+ int slice_iters; // number of threadblock tiles in the current slice
312
+ int slice_count =
313
+ 0; // total number of active threadblocks in the current slice
314
+ int slice_idx; // index of threadblock in current slice; numbered bottom to
315
+ // top
316
+
317
+ // We can easily implement parallel problem execution by just remapping
318
+ // indices and advancing global pointers
319
+ if (slice_col_par >= n_tiles) {
320
+ A += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_k / 8;
321
+ C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8;
322
+ locks += (slice_col_par / n_tiles) * n_tiles;
323
+ slice_col = slice_col_par % n_tiles;
324
+ }
325
+
326
+ // Compute all information about the current slice which is required for
327
+ // synchronization.
328
+ auto init_slice = [&]() {
329
+ slice_iters =
330
+ iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row);
331
+ if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0;
332
+ if (slice_iters == 0) return;
333
+ if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row;
334
+ slice_count = 1;
335
+ slice_idx = 0;
336
+ int col_first = iters * div_ceil(k_tiles * slice_col_par, iters);
337
+ if (col_first <= k_tiles * (slice_col_par + 1)) {
338
+ int col_off = col_first - k_tiles * slice_col_par;
339
+ slice_count = div_ceil(k_tiles - col_off, iters);
340
+ if (col_off > 0) slice_count++;
341
+ int delta_first = iters * blockIdx.x - col_first;
342
+ if (delta_first < 0 || (col_off == 0 && delta_first == 0))
343
+ slice_idx = slice_count - 1;
344
+ else {
345
+ slice_idx = slice_count - 1 - delta_first / iters;
346
+ if (col_off > 0) slice_idx--;
347
+ }
348
+ }
349
+ if (slice_col == n_tiles) {
350
+ A += 16 * thread_m_blocks * prob_k / 8;
351
+ C += 16 * thread_m_blocks * prob_n / 8;
352
+ locks += n_tiles;
353
+ slice_col = 0;
354
+ }
355
+ };
356
+ init_slice();
357
+
358
+ // A sizes/strides
359
+
360
+ // stride of the A matrix in global memory
361
+ int a_gl_stride = prob_k / 8;
362
+ // stride of an A matrix tile in shared memory
363
+ constexpr int a_sh_stride = 16 * thread_k_blocks / 8;
364
+ // delta between subsequent A tiles in global memory
365
+ constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8;
366
+ // between subsequent accesses within a tile
367
+ int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o);
368
+ // between shared memory writes
369
+ constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o);
370
+ // between shared memory tile reads
371
+ constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4));
372
+ // within a shared memory tile
373
+ constexpr int a_sh_rd_delta_i = a_sh_stride * 16;
374
+ // overall size of a tile
375
+ constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks);
376
+ // number of shared write iterations for a tile
377
+ constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta);
378
+
379
+ // B sizes/strides
380
+ int b_gl_stride = 16 * prob_n / (pack_factor * 4);
381
+ constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4;
382
+ constexpr int b_thread_vecs = num_bits == 4 ? 1 : 2;
383
+ constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs;
384
+
385
+ int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks;
386
+ int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads);
387
+ constexpr int b_sh_wr_delta = threads * b_thread_vecs;
388
+ constexpr int b_sh_rd_delta = threads * b_thread_vecs;
389
+ constexpr int b_sh_stage = b_sh_stride * thread_k_blocks;
390
+ constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta;
391
+
392
+ // Scale sizes/strides without act_order
393
+ int s_gl_stride = prob_n / 8;
394
+ constexpr int s_sh_stride = 16 * thread_n_blocks / 8;
395
+
396
+ // Scale size/strides with act_order
397
+ constexpr int tb_k = 16 * thread_k_blocks;
398
+ constexpr int g_idx_stage = 0;
399
+ // constexpr int act_s_row_stride = 1;
400
+ // int act_s_col_stride = act_s_row_stride * num_groups;
401
+ int act_s_col_stride = 1;
402
+ int act_s_col_warp_stride = act_s_col_stride * 8;
403
+ int tb_n_warps = thread_n_blocks / 4;
404
+ int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps;
405
+
406
+ // Global A read index of current thread.
407
+ int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
408
+ (threadIdx.x % a_gl_rd_delta_o);
409
+ a_gl_rd += a_gl_rd_delta_o * slice_row;
410
+ // Shared write index of current thread.
411
+ int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) +
412
+ (threadIdx.x % a_gl_rd_delta_o);
413
+ // Shared read index.
414
+ int a_sh_rd =
415
+ a_sh_stride * ((threadIdx.x % 32) % 16) + (threadIdx.x % 32) / 16;
416
+ a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4));
417
+
418
+ int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) +
419
+ (threadIdx.x % b_sh_stride_threads) * b_thread_vecs;
420
+ b_gl_rd += b_sh_stride * slice_col;
421
+ b_gl_rd += b_gl_rd_delta_o * slice_row;
422
+ int b_sh_wr = threadIdx.x * b_thread_vecs;
423
+ int b_sh_rd = threadIdx.x * b_thread_vecs;
424
+
425
+ // For act_order
426
+ int slice_k_start = tb_k * slice_row;
427
+ int slice_k_start_shared_fetch = slice_k_start;
428
+ int slice_n_offset = act_s_col_tb_stride * slice_col;
429
+
430
+ // No act_order
431
+ int s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
432
+ int s_sh_wr = threadIdx.x;
433
+ bool s_sh_wr_pred = threadIdx.x < s_sh_stride;
434
+
435
+ // We scale a `half2` tile in row-major layout for column-wise quantization.
436
+ int s_sh_rd =
437
+ 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) % 4;
438
+
439
+ // Precompute which thread should not read memory in which iterations; this is
440
+ // needed if there are more threads than required for a certain tilesize or
441
+ // when the batchsize is not a multiple of 16.
442
+ bool a_sh_wr_pred[a_sh_wr_iters];
443
+ #pragma unroll
444
+ for (int i = 0; i < a_sh_wr_iters; i++)
445
+ a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m;
446
+
447
+ // To ensure that writing and reading A tiles to/from shared memory, the
448
+ // latter in fragment format, is fully bank conflict free, we need to use a
449
+ // rather fancy XOR-based layout. The key here is that neither reads nor
450
+ // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the
451
+ // same shared memory banks. Further, it seems (based on NSight-Compute) that
452
+ // each warp must also write a consecutive memory segment?
453
+ auto transform_a = [&](int i) {
454
+ int row = i / a_gl_rd_delta_o;
455
+ return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row;
456
+ };
457
+ // Since the computation of this remapping is non-trivial and, due to our main
458
+ // loop unrolls, all shared memory accesses are static, we simply precompute
459
+ // both transformed reads and writes.
460
+ int a_sh_wr_trans[a_sh_wr_iters];
461
+ #pragma unroll
462
+ for (int i = 0; i < a_sh_wr_iters; i++)
463
+ a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr);
464
+ int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks];
465
+ #pragma unroll
466
+ for (int i = 0; i < b_sh_wr_iters; i++) {
467
+ #pragma unroll
468
+ for (int j = 0; j < thread_m_blocks; j++)
469
+ a_sh_rd_trans[i][j] =
470
+ transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd);
471
+ }
472
+
473
+ // Since B-accesses have non-constant stride they have to be computed at
474
+ // runtime; we break dependencies between subsequent accesses with a tile by
475
+ // maintining multiple pointers (we have enough registers), a tiny
476
+ // optimization.
477
+ const int4* B_ptr[b_sh_wr_iters];
478
+ #pragma unroll
479
+ for (int i = 0; i < b_sh_wr_iters; i++)
480
+ B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd;
481
+
482
+ extern __shared__ int4 sh[];
483
+ // Shared memory storage for global fetch pipelines.
484
+ int4* sh_a = sh;
485
+ int4* sh_b = sh_a + (stages * a_sh_stage);
486
+ int4* sh_g_idx = sh_b + (stages * b_sh_stage);
487
+ int4* sh_s = sh_g_idx + (stages * g_idx_stage);
488
+
489
+ // Register storage for double buffer of shared memory reads.
490
+ FragA frag_a[2][thread_m_blocks];
491
+ I4 frag_b_quant[2][b_thread_vecs];
492
+ FragC frag_c[thread_m_blocks][4][2];
493
+ FragS frag_s[2][4];
494
+
495
+ // Zero accumulators.
496
+ auto zero_accums = [&]() {
497
+ #pragma unroll
498
+ for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++)
499
+ reinterpret_cast<float*>(frag_c)[i] = 0;
500
+ };
501
+
502
+ int sh_first_group_id = -1;
503
+ int sh_num_groups = -1;
504
+ constexpr int sh_max_num_groups = 32;
505
+
506
+ auto fetch_scales_to_shared = [&](bool is_async, int first_group_id,
507
+ int last_group_id) {
508
+ sh_first_group_id = first_group_id;
509
+ sh_num_groups = last_group_id - first_group_id + 1;
510
+
511
+ if (sh_num_groups < sh_max_num_groups) {
512
+ sh_num_groups = sh_max_num_groups;
513
+ }
514
+
515
+ if (sh_first_group_id + sh_num_groups > num_groups) {
516
+ sh_num_groups = num_groups - sh_first_group_id;
517
+ }
518
+
519
+ int row_offset = first_group_id * s_gl_stride;
520
+
521
+ if (is_async) {
522
+ for (int i = 0; i < sh_num_groups; i++) {
523
+ if (threadIdx.x < s_sh_stride) {
524
+ cp_async4_pred(&sh_s[(i * s_sh_stride) + threadIdx.x],
525
+ &scales_ptr[row_offset + (i * s_gl_stride) +
526
+ slice_n_offset + threadIdx.x]);
527
+ }
528
+ }
529
+ } else {
530
+ for (int i = 0; i < sh_num_groups; i++) {
531
+ if (threadIdx.x < s_sh_stride) {
532
+ sh_s[(i * s_sh_stride) + threadIdx.x] =
533
+ scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset +
534
+ threadIdx.x];
535
+ }
536
+ }
537
+ }
538
+ };
539
+ // Asynchronously fetch the next A, B and s tile from global to the next
540
+ // shared memory pipeline location.
541
+ auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) {
542
+ if (pred) {
543
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
544
+ #pragma unroll
545
+ for (int i = 0; i < a_sh_wr_iters; i++) {
546
+ cp_async4_pred(
547
+ &sh_a_stage[a_sh_wr_trans[i]],
548
+ &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off],
549
+ a_sh_wr_pred[i]);
550
+ }
551
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
552
+ #pragma unroll
553
+ for (int i = 0; i < b_sh_wr_iters; i++) {
554
+ #pragma unroll
555
+ for (int j = 0; j < b_thread_vecs; j++) {
556
+ cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j);
557
+ }
558
+
559
+ B_ptr[i] += b_gl_rd_delta_o;
560
+ }
561
+ }
562
+ // Insert a fence even when we are winding down the pipeline to ensure that
563
+ // waiting is also correct at this point.
564
+ cp_async_fence();
565
+ };
566
+
567
+ // Wait until the next thread tile has been loaded to shared memory.
568
+ auto wait_for_stage = [&]() {
569
+ // We only have `stages - 2` active fetches since we are double buffering
570
+ // and can only issue the next fetch when it is guaranteed that the previous
571
+ // shared memory load is fully complete (as it may otherwise be
572
+ // overwritten).
573
+ cp_async_wait<stages - 2>();
574
+ __syncthreads();
575
+ };
576
+
577
+ // Load the next sub-tile from the current location in the shared memory pipe
578
+ // into the current register buffer.
579
+ auto fetch_to_registers = [&](int k, int pipe) {
580
+ int4* sh_a_stage = sh_a + a_sh_stage * pipe;
581
+ #pragma unroll
582
+ for (int i = 0; i < thread_m_blocks; i++)
583
+ ldsm4<scalar_t>(frag_a[k % 2][i],
584
+ &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]);
585
+ int4* sh_b_stage = sh_b + b_sh_stage * pipe;
586
+
587
+ #pragma unroll
588
+ for (int i = 0; i < b_thread_vecs; i++) {
589
+ frag_b_quant[k % 2][i] = *reinterpret_cast<I4*>(
590
+ &sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]);
591
+ }
592
+ };
593
+
594
+ bool is_same_group[stages];
595
+ int same_group_id[stages];
596
+
597
+ auto init_same_group = [&](int pipe) {
598
+ is_same_group[pipe] = false;
599
+ same_group_id[pipe] = 0;
600
+ return;
601
+ };
602
+
603
+ // Execute the actual tensor core matmul of a sub-tile.
604
+ auto matmul = [&](int k) {
605
+ // We have the m dimension as the inner loop in order to encourage overlapping
606
+ // dequantization and matmul operations.
607
+ #pragma unroll
608
+ for (int j = 0; j < 4; j++) {
609
+ FragB frag_b0;
610
+ FragB frag_b1;
611
+
612
+ int* frag_b_quant_ptr = reinterpret_cast<int*>(frag_b_quant[k % 2]);
613
+ int b_quant_0 = frag_b_quant_ptr[j * 2 + 0];
614
+ int b_quant_1 = frag_b_quant_ptr[j * 2 + 1];
615
+
616
+ frag_b0 = dequant_8bit<scalar_t>(b_quant_0);
617
+ frag_b1 = dequant_8bit<scalar_t>(b_quant_1);
618
+
619
+ #pragma unroll
620
+ for (int i = 0; i < thread_m_blocks; i++) {
621
+ mma<scalar_t>(frag_a[k % 2][i], frag_b0, frag_c[i][j][0]);
622
+ mma<scalar_t>(frag_a[k % 2][i], frag_b1, frag_c[i][j][1]);
623
+ }
624
+ }
625
+ };
626
+
627
+ // Since we slice across the k dimension of a tile in order to increase the
628
+ // number of warps while keeping the n dimension of a tile reasonable, we have
629
+ // multiple warps that accumulate their partial sums of the same output
630
+ // location; which we have to reduce over in the end. We do in shared memory.
631
+ auto thread_block_reduce = [&]() {
632
+ constexpr int red_off = threads / b_sh_stride_threads / 2;
633
+ if (red_off >= 1) {
634
+ int red_idx = threadIdx.x / b_sh_stride_threads;
635
+ constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2;
636
+ constexpr int red_sh_delta = b_sh_stride_threads;
637
+ int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) +
638
+ (threadIdx.x % b_sh_stride_threads);
639
+
640
+ // Parallel logarithmic shared memory reduction. We make sure to avoid any
641
+ // unnecessary read or write iterations, e.g., for two warps we write only
642
+ // once by warp 1 and read only once by warp 0.
643
+
644
+ #pragma unroll
645
+ for (int m_block = 0; m_block < thread_m_blocks; m_block++) {
646
+ #pragma unroll
647
+ for (int i = red_off; i > 0; i /= 2) {
648
+ if (i <= red_idx && red_idx < 2 * i) {
649
+ #pragma unroll
650
+ for (int j = 0; j < 4 * 2; j++) {
651
+ int red_sh_wr =
652
+ red_sh_delta * j + (red_sh_rd - red_sh_stride * i);
653
+ if (i < red_off) {
654
+ float* c_rd =
655
+ reinterpret_cast<float*>(&sh[red_sh_delta * j + red_sh_rd]);
656
+ float* c_wr = reinterpret_cast<float*>(&sh[red_sh_wr]);
657
+ #pragma unroll
658
+ for (int k = 0; k < 4; k++)
659
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + j][k] +=
660
+ c_rd[k] + c_wr[k];
661
+ }
662
+ sh[red_sh_wr] =
663
+ reinterpret_cast<int4*>(&frag_c)[4 * 2 * m_block + j];
664
+ }
665
+ }
666
+ __syncthreads();
667
+ }
668
+ if (red_idx == 0) {
669
+ #pragma unroll
670
+ for (int i = 0; i < 4 * 2; i++) {
671
+ float* c_rd =
672
+ reinterpret_cast<float*>(&sh[red_sh_delta * i + red_sh_rd]);
673
+ #pragma unroll
674
+ for (int j = 0; j < 4; j++)
675
+ reinterpret_cast<FragC*>(frag_c)[4 * 2 * m_block + i][j] +=
676
+ c_rd[j];
677
+ }
678
+ }
679
+ __syncthreads();
680
+ }
681
+ }
682
+ };
683
+
684
+ // Since multiple threadblocks may process parts of the same column slice, we
685
+ // finally have to globally reduce over the results. As the striped
686
+ // partitioning minimizes the number of such reductions and our outputs are
687
+ // usually rather small, we perform this reduction serially in L2 cache.
688
+ auto global_reduce = [&](bool first = false, bool last = false) {
689
+ // We are very careful here to reduce directly in the output buffer to
690
+ // maximize L2 cache utilization in this step. To do this, we write out
691
+ // results in FP16 (but still reduce with FP32 compute).
692
+ constexpr int active_threads = 32 * thread_n_blocks / 4;
693
+ if (threadIdx.x < active_threads) {
694
+ int c_gl_stride = prob_n / 8;
695
+ int c_gl_wr_delta_o = 8 * c_gl_stride;
696
+ int c_gl_wr_delta_i = 4 * (active_threads / 32);
697
+ int c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) +
698
+ 4 * (threadIdx.x / 32) + threadIdx.x % 4;
699
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
700
+ constexpr int c_sh_wr_delta = active_threads;
701
+ int c_sh_wr = threadIdx.x;
702
+
703
+ int row = (threadIdx.x % 32) / 4;
704
+
705
+ if (!first) {
706
+ // Interestingly, doing direct global accesses here really seems to mess up
707
+ // the compiler and lead to slowdowns, hence we also use async-copies even
708
+ // though these fetches are not actually asynchronous.
709
+ #pragma unroll
710
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
711
+ cp_async4_pred(
712
+ &sh[c_sh_wr + c_sh_wr_delta * i],
713
+ &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) +
714
+ c_gl_wr_delta_i * (i % 2)],
715
+ i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m);
716
+ }
717
+ cp_async_fence();
718
+ cp_async_wait<0>();
719
+ }
720
+
721
+ #pragma unroll
722
+ for (int i = 0; i < thread_m_blocks * 4; i++) {
723
+ if (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) {
724
+ if (!first) {
725
+ int4 c_red = sh[c_sh_wr + i * c_sh_wr_delta];
726
+ #pragma unroll
727
+ for (int j = 0; j < 2 * 4; j++) {
728
+ reinterpret_cast<float*>(
729
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)] +=
730
+ Dtype::num2float(reinterpret_cast<scalar_t*>(&c_red)[j]);
731
+ }
732
+ }
733
+ if (!last) {
734
+ int4 c;
735
+ #pragma unroll
736
+ for (int j = 0; j < 2 * 4; j++) {
737
+ reinterpret_cast<scalar_t*>(&c)[j] =
738
+ Dtype::float2num(reinterpret_cast<float*>(
739
+ &frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4)]);
740
+ }
741
+ C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] =
742
+ c;
743
+ }
744
+ }
745
+ }
746
+ }
747
+ };
748
+
749
+ // Write out the reduce final result in the correct layout. We only actually
750
+ // reshuffle matrix fragments in this step, the reduction above is performed
751
+ // in fragment layout.
752
+ auto write_result = [&]() {
753
+ int c_gl_stride = prob_n / 8;
754
+ constexpr int c_sh_stride = 2 * thread_n_blocks + 1;
755
+ int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks));
756
+ constexpr int c_sh_rd_delta =
757
+ c_sh_stride * (threads / (2 * thread_n_blocks));
758
+
759
+ int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) +
760
+ (threadIdx.x % (2 * thread_n_blocks));
761
+ c_gl_wr += (2 * thread_n_blocks) * slice_col;
762
+ int c_sh_wr =
763
+ (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4;
764
+ c_sh_wr += 32 * (threadIdx.x / 32);
765
+ int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) +
766
+ (threadIdx.x % (2 * thread_n_blocks));
767
+
768
+ int c_gl_wr_end = c_gl_stride * prob_m;
769
+
770
+ // We first reorder in shared memory to guarantee the most efficient final
771
+ // global write patterns
772
+ auto write = [&](int idx, float c0, float c1, FragS& s) {
773
+ scalar_t2 res =
774
+ Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1));
775
+
776
+ ((scalar_t2*)sh)[idx] = res;
777
+ };
778
+
779
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
780
+ #pragma unroll
781
+ for (int i = 0; i < thread_m_blocks; i++) {
782
+ #pragma unroll
783
+ for (int j = 0; j < 4; j++) {
784
+ int wr = c_sh_wr + 8 * j;
785
+ write(wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0],
786
+ frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]);
787
+ write(wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2],
788
+ frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]);
789
+ write(wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0],
790
+ frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]);
791
+ write(wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2],
792
+ frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]);
793
+ }
794
+ c_sh_wr += 16 * (4 * c_sh_stride);
795
+ }
796
+ }
797
+ __syncthreads();
798
+
799
+ #pragma unroll
800
+ for (int i = 0;
801
+ i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks));
802
+ i++) {
803
+ if (c_gl_wr < c_gl_wr_end) {
804
+ C[c_gl_wr] = sh[c_sh_rd];
805
+ c_gl_wr += c_gl_wr_delta;
806
+ c_sh_rd += c_sh_rd_delta;
807
+ }
808
+ }
809
+ };
810
+
811
+ // Start global fetch and register load pipelines.
812
+ auto start_pipes = [&]() {
813
+
814
+ #pragma unroll
815
+ for (int i = 0; i < stages - 1; i++) {
816
+ fetch_to_shared(i, i, i < slice_iters);
817
+ }
818
+
819
+ zero_accums();
820
+ wait_for_stage();
821
+ init_same_group(0);
822
+ fetch_to_registers(0, 0);
823
+ a_gl_rd += a_gl_rd_delta_o * (stages - 1);
824
+ slice_k_start_shared_fetch += tb_k * (stages - 1);
825
+ };
826
+ if (slice_iters) {
827
+ start_pipes();
828
+ }
829
+
830
+ // Main loop.
831
+ while (slice_iters) {
832
+ // We unroll over both the global fetch and the register load pipeline to
833
+ // ensure all shared memory accesses are static. Note that both pipelines
834
+ // have even length meaning that the next iteration will always start at
835
+ // index 0.
836
+
837
+ #pragma unroll
838
+ for (int pipe = 0; pipe < stages;) {
839
+ #pragma unroll
840
+ for (int k = 0; k < b_sh_wr_iters; k++) {
841
+ fetch_to_registers(k + 1, pipe % stages);
842
+ if (k == b_sh_wr_iters - 2) {
843
+ fetch_to_shared((pipe + stages - 1) % stages, pipe,
844
+ slice_iters >= stages);
845
+ pipe++;
846
+ wait_for_stage();
847
+ init_same_group(pipe % stages);
848
+ }
849
+ matmul(k);
850
+ }
851
+ slice_iters--;
852
+ if (slice_iters == 0) {
853
+ break;
854
+ }
855
+ }
856
+
857
+ a_gl_rd += a_gl_rd_delta_o * stages;
858
+ slice_k_start += tb_k * stages;
859
+ slice_k_start_shared_fetch += tb_k * stages;
860
+
861
+ // Process results and, if necessary, proceed to the next column slice.
862
+ // While this pattern may not be the most readable, other ways of writing
863
+ // the loop seemed to noticeably worse performance after compilation.
864
+ if (slice_iters == 0) {
865
+ cp_async_wait<0>();
866
+ bool last = slice_idx == slice_count - 1;
867
+ // For per-column scales, we only fetch them here in the final step before
868
+ // write-out
869
+ if (s_sh_wr_pred) {
870
+ cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]);
871
+ }
872
+ cp_async_fence();
873
+
874
+ thread_block_reduce();
875
+
876
+ cp_async_wait<0>();
877
+ __syncthreads();
878
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
879
+ reinterpret_cast<int4*>(&frag_s)[0] = sh_s[s_sh_rd + 0];
880
+ reinterpret_cast<int4*>(&frag_s)[1] = sh_s[s_sh_rd + 4];
881
+ }
882
+
883
+ // For 8-bit channelwise, we apply the scale before the global reduction
884
+ // that converts the fp32 results to fp16 (so that we avoid possible
885
+ // overflow in fp16)
886
+ if (threadIdx.x / 32 < thread_n_blocks / 4) {
887
+ #pragma unroll
888
+ for (int i = 0; i < thread_m_blocks; i++) {
889
+ #pragma unroll
890
+ for (int j = 0; j < 4; j++) {
891
+ scale_float<scalar_t>(reinterpret_cast<float*>(&frag_c[i][j][0][0]),
892
+ frag_s[j / 2][2 * (j % 2) + 0]);
893
+ scale_float<scalar_t>(reinterpret_cast<float*>(&frag_c[i][j][0][2]),
894
+ frag_s[j / 2][2 * (j % 2) + 0]);
895
+
896
+ scale_float<scalar_t>(reinterpret_cast<float*>(&frag_c[i][j][1][0]),
897
+ frag_s[j / 2][2 * (j % 2) + 1]);
898
+ scale_float<scalar_t>(reinterpret_cast<float*>(&frag_c[i][j][1][2]),
899
+ frag_s[j / 2][2 * (j % 2) + 1]);
900
+ }
901
+ }
902
+ }
903
+
904
+ if (slice_count > 1) { // only globally reduce if there is more than one
905
+ // block in a slice
906
+ barrier_acquire(&locks[slice_col], slice_idx);
907
+ global_reduce(slice_idx == 0, last);
908
+ barrier_release(&locks[slice_col], last);
909
+ }
910
+ if (last) // only the last block in a slice actually writes the result
911
+ write_result();
912
+ slice_row = 0;
913
+ slice_col_par++;
914
+ slice_col++;
915
+ init_slice();
916
+ if (slice_iters) {
917
+ a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) +
918
+ (threadIdx.x % a_gl_rd_delta_o);
919
+ #pragma unroll
920
+ for (int i = 0; i < b_sh_wr_iters; i++)
921
+ B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles;
922
+ if (slice_col == 0) {
923
+ #pragma unroll
924
+ for (int i = 0; i < b_sh_wr_iters; i++) B_ptr[i] -= b_gl_stride;
925
+ }
926
+
927
+ // Update slice k/n for scales loading
928
+ s_gl_rd = s_sh_stride * slice_col + threadIdx.x;
929
+
930
+ start_pipes();
931
+ }
932
+ }
933
+ }
934
+}
935
+
936
+ #define __CALL_IF(NUM_BITS, THREAD_M_BLOCKS, THREAD_N_BLOCKS, \
937
+ THREAD_K_BLOCKS, GROUP_BLOCKS, NUM_THREADS) \
938
+ else if (num_bits == NUM_BITS && thread_m_blocks == THREAD_M_BLOCKS && \
939
+ thread_n_blocks == THREAD_N_BLOCKS && \
940
+ thread_k_blocks == THREAD_K_BLOCKS && \
941
+ group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS) { \
942
+ cudaFuncSetAttribute( \
943
+ Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
944
+ THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, GROUP_BLOCKS>, \
945
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
946
+ Marlin<scalar_t, NUM_BITS, NUM_THREADS, THREAD_M_BLOCKS, \
947
+ THREAD_N_BLOCKS, THREAD_K_BLOCKS, pipe_stages, GROUP_BLOCKS> \
948
+ <<<blocks, NUM_THREADS, max_shared_mem, stream>>>( \
949
+ A_ptr, B_ptr, C_ptr, s_ptr, num_groups, prob_m, prob_n, prob_k, \
950
+ locks); \
951
+ }
952
+
953
+typedef struct {
954
+ int thread_k;
955
+ int thread_n;
956
+ int num_threads;
957
+} thread_config_t;
958
+
959
+typedef struct {
960
+ int max_m_blocks;
961
+ thread_config_t tb_cfg;
962
+} exec_config_t;
963
+
964
+thread_config_t small_batch_thread_configs[] = {
965
+ // Ordered by priority
966
+
967
+ // thread_k, thread_n, num_threads
968
+ {128, 128, 256},
969
+ {64, 128, 128},
970
+ {128, 64, 128},
971
+};
972
+
973
+thread_config_t large_batch_thread_configs[] = {
974
+ // Ordered by priority
975
+
976
+ // thread_k, thread_n, num_threads
977
+ {64, 256, 256},
978
+ {64, 128, 128},
979
+ {128, 64, 128},
980
+
981
+};
982
+
983
+int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
984
+ int prob_n, int prob_k, int num_bits,
985
+ int group_size) {
986
+ int tb_n = th_config.thread_n;
987
+
988
+ // Get max scale groups per thread-block
989
+ // Fixed for channelwise
990
+ int tb_groups = 1;
991
+ int tb_scales = tb_groups * tb_n * 2;
992
+
993
+ return tb_scales * pipe_stages;
994
+}
995
+
996
+bool is_valid_cache_size(thread_config_t const& th_config, int max_m_blocks,
997
+ int prob_m, int prob_n, int prob_k, int num_bits,
998
+ int scales_cache_size, int max_shared_mem) {
999
+ int pack_factor = 32 / num_bits;
1000
+
1001
+ // Get B size
1002
+ int tb_k = th_config.thread_k;
1003
+ int tb_n = th_config.thread_n;
1004
+
1005
+ int b_size = (tb_k * tb_n / pack_factor) * 4;
1006
+
1007
+ // Get A size
1008
+ int m_blocks = div_ceil(prob_m, 16);
1009
+ int tb_max_m = 16;
1010
+
1011
+ while (true) {
1012
+ if (m_blocks >= max_m_blocks) {
1013
+ tb_max_m *= max_m_blocks;
1014
+ break;
1015
+ }
1016
+
1017
+ max_m_blocks--;
1018
+ if (max_m_blocks == 0) {
1019
+ TORCH_CHECK(false, "Unexpected m_blocks = ", m_blocks);
1020
+ }
1021
+ }
1022
+
1023
+ int a_size = (tb_max_m * tb_k) * 2;
1024
+
1025
+ float pipe_size = (a_size + b_size) * pipe_stages;
1026
+
1027
+ TORCH_CHECK(max_shared_mem / 2 > scales_cache_size); // Sanity
1028
+
1029
+ return pipe_size < 0.95f * (max_shared_mem - scales_cache_size);
1030
+}
1031
+
1032
+bool is_valid_config(thread_config_t const& th_config, int max_m_blocks,
1033
+ int prob_m, int prob_n, int prob_k, int num_bits,
1034
+ int group_size, int max_shared_mem) {
1035
+ // Sanity
1036
+ if (th_config.thread_k == -1 || th_config.thread_n == -1 ||
1037
+ th_config.num_threads == -1) {
1038
+ return false;
1039
+ }
1040
+
1041
+ // Verify K/N are divisible by thread K/N
1042
+ if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) {
1043
+ return false;
1044
+ }
1045
+
1046
+ // Verify min for thread K/N
1047
+ if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) {
1048
+ return false;
1049
+ }
1050
+
1051
+ // num_threads must be at least 128 (= 4 warps)
1052
+ if (th_config.num_threads < 128) {
1053
+ return false;
1054
+ }
1055
+
1056
+ // Determine cache for scales
1057
+ int scales_cache_size = get_scales_cache_size(th_config, prob_m, prob_n,
1058
+ prob_k, num_bits, group_size);
1059
+
1060
+ // Check that pipeline fits into cache
1061
+ if (!is_valid_cache_size(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1062
+ num_bits, scales_cache_size, max_shared_mem)) {
1063
+ return false;
1064
+ }
1065
+
1066
+ return true;
1067
+}
1068
+
1069
+exec_config_t determine_thread_config(int prob_m, int prob_n, int prob_k,
1070
+ int num_bits, int group_size,
1071
+ int max_shared_mem) {
1072
+ int max_m_blocks = 4;
1073
+ while (max_m_blocks > 0) {
1074
+ if (prob_m <= 16) {
1075
+ for (auto th_config : small_batch_thread_configs) {
1076
+ if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1077
+ num_bits, group_size, max_shared_mem)) {
1078
+ return exec_config_t{max_m_blocks, th_config};
1079
+ }
1080
+ }
1081
+ } else {
1082
+ for (auto th_config : large_batch_thread_configs) {
1083
+ if (is_valid_config(th_config, max_m_blocks, prob_m, prob_n, prob_k,
1084
+ num_bits, group_size, max_shared_mem)) {
1085
+ return exec_config_t{max_m_blocks, th_config};
1086
+ }
1087
+ }
1088
+ }
1089
+
1090
+ max_m_blocks--; // Process less M blocks per invocation to reduce cache
1091
+ // usage
1092
+ }
1093
+
1094
+ return exec_config_t{0, {-1, -1, -1}};
1095
+}
1096
+
1097
+ #define CALL_IF(NUM_BITS, N_BLOCKS, K_BLOCKS, NUM_THREADS) \
1098
+ __CALL_IF(NUM_BITS, 1, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1099
+ __CALL_IF(NUM_BITS, 2, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1100
+ __CALL_IF(NUM_BITS, 3, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS) \
1101
+ __CALL_IF(NUM_BITS, 4, N_BLOCKS, K_BLOCKS, -1, NUM_THREADS)
1102
+
1103
+template <typename scalar_t>
1104
+void marlin_mm_f16i4(const void* A, const void* B, void* C, void* s, int prob_m,
1105
+ int prob_n, int prob_k, void* workspace, int num_bits,
1106
+ int num_groups, int group_size, int dev,
1107
+ cudaStream_t stream, int thread_k, int thread_n, int sms,
1108
+ int max_par) {
1109
+ TORCH_CHECK(num_bits == 8, "num_bits must be 8. Got = ", num_bits);
1110
+ TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m,
1111
+ ", ", prob_n, ", ", prob_k, "]");
1112
+
1113
+ int tot_m = prob_m;
1114
+ int tot_m_blocks = div_ceil(tot_m, 16);
1115
+ int pad = 16 * tot_m_blocks - tot_m;
1116
+
1117
+ if (sms == -1) {
1118
+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
1119
+ }
1120
+
1121
+ int max_shared_mem = 0;
1122
+ cudaDeviceGetAttribute(&max_shared_mem,
1123
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
1124
+ TORCH_CHECK(max_shared_mem > 0);
1125
+
1126
+ // Set thread config
1127
+ exec_config_t exec_cfg;
1128
+ if (thread_k != -1 && thread_n != -1) {
1129
+ // User-defined config
1130
+ exec_cfg =
1131
+ exec_config_t{4, thread_config_t{thread_k, thread_n, default_threads}};
1132
+ } else {
1133
+ // Auto config
1134
+ exec_cfg = determine_thread_config(prob_m, prob_n, prob_k, num_bits,
1135
+ group_size, max_shared_mem);
1136
+ }
1137
+
1138
+ TORCH_CHECK(
1139
+ exec_cfg.max_m_blocks > 0 &&
1140
+ is_valid_config(exec_cfg.tb_cfg, exec_cfg.max_m_blocks, prob_m,
1141
+ prob_n, prob_k, num_bits, group_size, max_shared_mem),
1142
+ "Invalid thread config: max_m_blocks = ", exec_cfg.max_m_blocks,
1143
+ ", thread_k = ", exec_cfg.tb_cfg.thread_k,
1144
+ ", thread_n = ", exec_cfg.tb_cfg.thread_n,
1145
+ ", num_threads = ", exec_cfg.tb_cfg.num_threads, " for MKN = [", prob_m,
1146
+ ", ", prob_k, ", ", prob_n, "] and num_bits = ", num_bits,
1147
+ ", group_size = ", group_size, ", max_shared_mem = ", max_shared_mem);
1148
+
1149
+ int num_threads = exec_cfg.tb_cfg.num_threads;
1150
+ thread_k = exec_cfg.tb_cfg.thread_k;
1151
+ thread_n = exec_cfg.tb_cfg.thread_n;
1152
+
1153
+ int thread_k_blocks = thread_k / 16;
1154
+ int thread_n_blocks = thread_n / 16;
1155
+
1156
+ int blocks = sms;
1157
+
1158
+ TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n,
1159
+ " is not divisible by thread_n = ", thread_n);
1160
+ TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k,
1161
+ " is not divisible by thread_k = ", thread_k);
1162
+
1163
+ int group_blocks = -1;
1164
+
1165
+ const int4* A_ptr = (const int4*)A;
1166
+ const int4* B_ptr = (const int4*)B;
1167
+ int4* C_ptr = (int4*)C;
1168
+ const int4* s_ptr = (const int4*)s;
1169
+
1170
+ int* locks = (int*)workspace;
1171
+
1172
+ // Main loop
1173
+ for (int i = 0; i < tot_m_blocks; i += exec_cfg.max_m_blocks) {
1174
+ int thread_m_blocks = tot_m_blocks - i;
1175
+ prob_m = tot_m - 16 * i;
1176
+ int par = 1;
1177
+ if (thread_m_blocks > exec_cfg.max_m_blocks) {
1178
+ // Note that parallel > 1 currently only works for inputs without any
1179
+ // padding
1180
+ par = (16 * thread_m_blocks - pad) / (16 * exec_cfg.max_m_blocks);
1181
+ if (par > max_par) par = max_par;
1182
+ prob_m = (16 * exec_cfg.max_m_blocks) * par;
1183
+ i += exec_cfg.max_m_blocks * (par - 1);
1184
+ thread_m_blocks = exec_cfg.max_m_blocks;
1185
+ }
1186
+
1187
+ // Define kernel configurations
1188
+ if (false) {
1189
+ }
1190
+ CALL_IF(8, 32, 2, 256)
1191
+ CALL_IF(8, 16, 4, 256)
1192
+ CALL_IF(8, 8, 8, 256)
1193
+ CALL_IF(8, 8, 4, 128)
1194
+ CALL_IF(8, 4, 8, 128)
1195
+ else {
1196
+ TORCH_CHECK(false, "Unsupported shapes: MNK = [" + str(prob_m) + ", " +
1197
+ str(prob_n) + ", " + str(prob_k) + "]" +
1198
+ ", num_groups = " + str(num_groups) +
1199
+ ", group_size = " + str(group_size) +
1200
+ ", thread_m_blocks = " + str(thread_m_blocks) +
1201
+ ", thread_n_blocks = " + str(thread_n_blocks) +
1202
+ ", thread_k_blocks = " + str(thread_k_blocks));
1203
+ }
1204
+
1205
+ A_ptr += 16 * thread_m_blocks * (prob_k / 8) * par;
1206
+ C_ptr += 16 * thread_m_blocks * (prob_n / 8) * par;
1207
+ }
1208
+}
1209
+
1210
+} // namespace fp8_marlin
1211
+
1212
+torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1213
+ torch::Tensor& b_scales, torch::Tensor& workspace,
1214
+ int64_t num_bits, int64_t size_m, int64_t size_n,
1215
+ int64_t size_k) {
1216
+ // Verify num_bits
1217
+ TORCH_CHECK(num_bits == 8, "num_bits must be 8. Got = ", num_bits);
1218
+ int pack_factor = 32 / num_bits;
1219
+
1220
+ // Verify A
1221
+ TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
1222
+ ", size_m = ", size_m);
1223
+ TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1),
1224
+ ", size_k = ", size_k);
1225
+
1226
+ // Verify B
1227
+ TORCH_CHECK(size_k % gptq_marlin::tile_size == 0, "size_k = ", size_k,
1228
+ " is not divisible by tile_size = ", gptq_marlin::tile_size);
1229
+ TORCH_CHECK((size_k / gptq_marlin::tile_size) == b_q_weight.size(0),
1230
+ "Shape mismatch: b_q_weight.size(0) = ", b_q_weight.size(0),
1231
+ ", size_k = ", size_k, ", tile_size = ", gptq_marlin::tile_size);
1232
+ TORCH_CHECK(b_q_weight.size(1) % gptq_marlin::tile_size == 0,
1233
+ "b_q_weight.size(1) = ", b_q_weight.size(1),
1234
+ " is not divisible by tile_size = ", gptq_marlin::tile_size);
1235
+ int actual_size_n =
1236
+ (b_q_weight.size(1) / gptq_marlin::tile_size) * pack_factor;
1237
+ TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n,
1238
+ ", actual_size_n = ", actual_size_n);
1239
+
1240
+ // Verify device and strides
1241
+ TORCH_CHECK(a.device().is_cuda(), "A is not on GPU");
1242
+ TORCH_CHECK(a.is_contiguous(), "A is not contiguous");
1243
+
1244
+ TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU");
1245
+ TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous");
1246
+
1247
+ TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU");
1248
+ TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous");
1249
+
1250
+ // Alloc buffers
1251
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
1252
+ auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device());
1253
+ torch::Tensor c = torch::empty({size_m, size_n}, options);
1254
+
1255
+ // thread_k: `k` size of a thread_tile in `weights` (can usually be left as
1256
+ // auto -1)
1257
+ int thread_k = -1;
1258
+ // thread_n: `n` size of a thread_tile in `weights` (can usually be left as
1259
+ // auto -1)
1260
+ int thread_n = -1;
1261
+ // sms: number of SMs to use for the kernel (can usually be left as auto -1)
1262
+ int sms = -1;
1263
+
1264
+ // Detect groupsize and act_order
1265
+ int num_groups = -1;
1266
+ int group_size = -1;
1267
+
1268
+ int b_rank = b_scales.sizes().size();
1269
+ TORCH_CHECK(b_rank == 2, "b_scales rank = ", b_rank, " is not 2");
1270
+ TORCH_CHECK(b_scales.size(1) == size_n, "b_scales dim 1 = ", b_scales.size(1),
1271
+ " is not size_n = ", size_n);
1272
+ // Channelwise only for FP8
1273
+ TORCH_CHECK(b_scales.size(0) == 1)
1274
+ num_groups = b_scales.size(0);
1275
+
1276
+ // Verify workspace size
1277
+ TORCH_CHECK(
1278
+ size_n % gptq_marlin::min_thread_n == 0, "size_n = ", size_n,
1279
+ ", is not divisible by min_thread_n = ", gptq_marlin::min_thread_n);
1280
+ int min_workspace_size =
1281
+ (size_n / gptq_marlin::min_thread_n) * gptq_marlin::max_par;
1282
+ TORCH_CHECK(workspace.numel() >= min_workspace_size,
1283
+ "workspace.numel = ", workspace.numel(),
1284
+ " is below min_workspace_size = ", min_workspace_size);
1285
+
1286
+ int dev = a.get_device();
1287
+ if (a.scalar_type() == at::ScalarType::Half) {
1288
+ fp8_marlin::marlin_mm_f16i4<half>(
1289
+ a.data_ptr<at::Half>(), b_q_weight.data_ptr(), c.data_ptr<at::Half>(),
1290
+ b_scales.data_ptr<at::Half>(), size_m, size_n, size_k,
1291
+ workspace.data_ptr(), num_bits, num_groups, group_size, dev,
1292
+ at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1293
+ gptq_marlin::max_par);
1294
+ } else if (a.scalar_type() == at::ScalarType::BFloat16) {
1295
+ fp8_marlin::marlin_mm_f16i4<nv_bfloat16>(
1296
+ a.data_ptr<at::BFloat16>(), b_q_weight.data_ptr(),
1297
+ c.data_ptr<at::BFloat16>(), b_scales.data_ptr<at::BFloat16>(), size_m,
1298
+ size_n, size_k, workspace.data_ptr(), num_bits, num_groups, group_size,
1299
+ dev, at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms,
1300
+ gptq_marlin::max_par);
1301
+ } else {
1302
+ TORCH_CHECK(false, "fp8_marlin_gemm only supports bfloat16 and float16");
1303
+ }
1304
+
1305
+ return c;
1306
+}
1307
+
1308
+#endif
csrc/torch_bindings.cpp
+4
@@ -137,6 +137,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
137
ops.def("gptq_marlin_repack", &gptq_marlin_repack);
138
ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
139
140
+ // fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
141
+ ops.def("fp8_marlin_gemm", &fp8_marlin_gemm);
142
+ ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
143
+
144
// CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
145
// quantization.
146
ops.def(
docs/source/quantization/fp8.rst
+2
-1
@@ -4,7 +4,8 @@ FP8
4
==================
5
6
vLLM supports FP8 (8-bit floating point) weight and activation quantization using hardware acceleration on GPUs such as Nvidia H100 and AMD MI300x.
7
-Currently, only Hopper and Ada Lovelace GPUs are supported.
7
+Currently, only Hopper and Ada Lovelace GPUs are officially supported for W8A8.
8
+Ampere GPUs are supported for W8A16 (weight-only FP8) utilizing Marlin kernels.
9
Quantization of models with FP8 allows for a 2x reduction in model memory requirements and up to a 1.6x improvement in throughput with minimal impact on accuracy.
10
11
Please visit the HF collection of `quantized FP8 checkpoints of popular LLMs ready to use with vLLM <https://huggingface.co/collections/neuralmagic/fp8-llms-for-vllm-666742ed2b78b7ac8df13127>`_.
docs/source/quantization/supported_hardware.rst
+1
-1
@@ -11,7 +11,7 @@ Implementation Volta Turing Ampere Ada Hopper AMD GPU Intel GPU x86
11
AQLM ✅ ✅ ✅ ✅ ✅ ❌ ❌ ❌ ❌ ❌
12
AWQ ❌ ✅ ✅ ✅ ✅ ❌ ❌ ❌ ❌ ❌
13
DeepSpeedFP ✅ ✅ ✅ ✅ ✅ ❌ ❌ ❌ ❌ ❌
14
-FP8 ❌ ❌ ❌ ✅ ✅ ❌ ❌ ❌ ❌ ❌
14
+FP8 ❌ ❌ ✅ ✅ ✅ ❌ ❌ ❌ ❌ ❌
15
Marlin ❌ ❌ ✅ ✅ ✅ ❌ ❌ ❌ ❌ ❌
16
GPTQ ✅ ✅ ✅ ✅ ✅ ❌ ❌ ❌ ❌ ❌
17
SqueezeLLM ✅ ✅ ✅ ✅ ✅ ❌ ❌ ❌ ❌ ❌
tests/kernels/test_marlin_gemm.py
+84
-4
@@ -8,7 +8,8 @@ import torch
8
from vllm import _custom_ops as ops
9
from vllm.model_executor.layers.quantization.gptq_marlin import (
10
GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
11
- GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
11
+ GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS,
12
+ marlin_permute_scales)
13
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
14
GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
15
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
@@ -16,7 +17,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
17
marlin_perm)
18
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
19
MarlinWorkspace, compute_max_diff, is_marlin_supported, marlin_24_quantize,
19
- marlin_quantize, marlin_weights)
20
+ marlin_quantize, marlin_weights, pack_fp8_to_int32)
21
from vllm.model_executor.layers.quantization.utils.quant_utils import (
22
gptq_pack, quantize_weights, sort_weights)
23
@@ -38,9 +39,11 @@ MNK_FACTORS = [
39
(67, 13, 11),
40
]
41
42
+DTYPES = [torch.float16, torch.bfloat16]
43
42
-def rand_data(shape):
43
- return torch.randn(shape, dtype=torch.half, device="cuda")
44
+
45
+def rand_data(shape, dtype=torch.float16):
46
+ return torch.randn(shape, dtype=dtype, device="cuda")
47
48
49
@pytest.mark.skipif(not is_marlin_supported(),
@@ -217,3 +220,80 @@ def test_marlin_24_gemm(k_chunk, n_chunk, num_bits, group_size, mnk_factors):
220
print("max_diff = {}".format(max_diff))
221
222
assert max_diff < 0.04
223
+
224
+
225
+@pytest.mark.skipif(not is_marlin_supported(),
226
+ reason="Marlin is not supported on this GPU type.")
227
+@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
228
+@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
229
+@pytest.mark.parametrize("num_bits", [8])
230
+@pytest.mark.parametrize("group_size", [-1])
231
+@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
232
+@pytest.mark.parametrize("dtype", DTYPES)
233
+def test_fp8_marlin_gemm(
234
+ k_chunk,
235
+ n_chunk,
236
+ num_bits,
237
+ group_size,
238
+ mnk_factors,
239
+ dtype,
240
+):
241
+ m_factor, n_factor, k_factor = mnk_factors
242
+
243
+ size_m = m_factor
244
+ size_k = k_chunk * k_factor
245
+ size_n = n_chunk * n_factor
246
+
247
+ print(f"MNK = {size_m} {size_n} {size_k}")
248
+ print(f"groupsize = {group_size}")
249
+
250
+ a_input = rand_data((size_m, size_k), dtype=dtype)
251
+ b_weight = rand_data((size_k, size_n), dtype=dtype)
252
+
253
+ # WEIGHTS
254
+ fp8_weight, weight_scale = ops.scaled_fp8_quant(b_weight, scale=None)
255
+ # Repack weights to gptq format (packed int32 elements)
256
+ packed_gptq_qweight = pack_fp8_to_int32(fp8_weight)
257
+ # Repack weights to marlin format
258
+ marlin_qweight = ops.gptq_marlin_repack(
259
+ b_q_weight=packed_gptq_qweight,
260
+ perm=torch.empty(0, dtype=torch.int, device="cuda"),
261
+ size_k=size_k,
262
+ size_n=size_n,
263
+ num_bits=8,
264
+ )
265
+
266
+ # WEIGHT SCALES
267
+ # Currently Marlin doesn't support per-tensor scales, so we
268
+ # expand it to channelwise
269
+ scales = weight_scale.repeat(1, size_n).to(a_input.dtype).to("cuda")
270
+ # Permute scales
271
+ marlin_scales = marlin_permute_scales(
272
+ s=scales,
273
+ size_k=size_k,
274
+ size_n=size_n,
275
+ group_size=-1,
276
+ num_bits=8,
277
+ )
278
+
279
+ workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
280
+ GPTQ_MARLIN_MAX_PARALLEL)
281
+
282
+ output = ops.fp8_marlin_gemm(
283
+ a=a_input,
284
+ b_q_weight=marlin_qweight,
285
+ b_scales=marlin_scales,
286
+ workspace=workspace.scratch,
287
+ num_bits=num_bits,
288
+ size_m=a_input.shape[0],
289
+ size_n=b_weight.shape[1],
290
+ size_k=a_input.shape[1],
291
+ )
292
+ output_ref = torch.matmul(a_input, b_weight)
293
+
294
+ torch.cuda.synchronize()
295
+
296
+ max_diff = compute_max_diff(output, output_ref)
297
+ print("max_diff = {}".format(max_diff))
298
+
299
+ assert max_diff < 0.04
tests/quantization/test_fp8.py
+14
-5
@@ -6,7 +6,7 @@ import pytest
6
import torch
7
8
from tests.quantization.utils import is_quant_method_supported
9
-from vllm._custom_ops import scaled_fp8_quant
9
+from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
11
12
MODELS = [
@@ -35,7 +35,16 @@ def test_load_fp16_model(vllm_runner) -> None:
35
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
36
fc1 = model.model.decoder.layers[0].fc1
37
assert isinstance(fc1.quant_method, Fp8LinearMethod)
38
- assert fc1.weight.dtype == torch.float8_e4m3fn
38
+
39
+ capability = torch.cuda.get_device_capability()
40
+ capability = capability[0] * 10 + capability[1]
41
+ if capability >= 89:
42
+ # For GPUs with hardware support, we keep weights in fp8
43
+ assert fc1.weight.dtype == torch.float8_e4m3fn
44
+ else:
45
+ # For GPUs without hardware support, we pack the fp8 weights
46
+ # for weight-only quantization using Marlin kernels
47
+ assert fc1.weight.dtype == torch.int32
48
49
50
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
@@ -63,7 +72,7 @@ def test_scaled_fp8_quant(dtype) -> None:
72
x = (torch.randn(size=(11, 11), device="cuda") * 13).to(dtype)
73
74
# Dynamic quantization
66
- ref_y, inv_scale = scaled_fp8_quant(x, None)
75
+ ref_y, inv_scale = ops.scaled_fp8_quant(x, None)
76
ref_y = per_tensor_dequantize(ref_y, inv_scale, dtype)
77
78
# Reference dynamic quantizaton
@@ -71,11 +80,11 @@ def test_scaled_fp8_quant(dtype) -> None:
80
assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
81
82
# Static quantization
74
- y, _ = scaled_fp8_quant(x, inv_scale)
83
+ y, _ = ops.scaled_fp8_quant(x, inv_scale)
84
assert torch.allclose(ref_y, per_tensor_dequantize(y, inv_scale, dtype))
85
86
# Padding
78
- y, _ = scaled_fp8_quant(x, inv_scale, batch_dim_padding=17)
87
+ y, _ = ops.scaled_fp8_quant(x, inv_scale, batch_dim_padding=17)
88
assert y.shape[0] == 17
89
assert torch.allclose(
90
ref_y,
vllm/_custom_ops.py
+9
@@ -271,6 +271,15 @@ def gptq_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
271
size_k, is_k_full)
272
273
274
+# fp8 marlin
275
+def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
276
+ b_scales: torch.Tensor, workspace: torch.Tensor,
277
+ num_bits: int, size_m: int, size_n: int,
278
+ size_k: int) -> torch.Tensor:
279
+ return torch.ops._C.fp8_marlin_gemm(a, b_q_weight, b_scales, workspace,
280
+ num_bits, size_m, size_n, size_k)
281
+
282
+
283
# fp8
284
def scaled_fp8_quant(
285
input: torch.Tensor,
vllm/model_executor/layers/quantization/fp8.py
+134
-30
@@ -11,6 +11,11 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
11
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
14
+from vllm.model_executor.layers.quantization.gptq_marlin import (
15
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQMarlinState,
16
+ marlin_permute_scales)
17
+from vllm.model_executor.layers.quantization.utils.marlin_utils import (
18
+ pack_fp8_to_int32)
19
from vllm.model_executor.utils import set_weight_attrs
20
from vllm.platforms import current_platform
21
from vllm.utils import print_warning_once
@@ -54,7 +59,7 @@ class Fp8Config(QuantizationConfig):
59
60
@classmethod
61
def get_min_capability(cls) -> int:
57
- return 89
62
+ return 80
63
64
@classmethod
65
def get_config_filenames(cls) -> List[str]:
@@ -106,6 +111,12 @@ class Fp8LinearMethod(LinearMethodBase):
111
self.quant_config = quant_config
112
self.cutlass_fp8_supported = cutlass_fp8_supported()
113
114
+ # For GPUs that lack FP8 hardware support, we can leverage the Marlin
115
+ # kernel for fast weight-only FP8 quantization
116
+ capability = current_platform.get_device_capability()
117
+ capability = capability[0] * 10 + capability[1]
118
+ self.use_marlin = capability < 89
119
+
120
def _create_scale_param(
121
self,
122
scale_name: str,
@@ -139,6 +150,10 @@ class Fp8LinearMethod(LinearMethodBase):
150
layer.process_after_load = True
151
layer.logical_widths = output_partition_sizes
152
153
+ layer.input_size_per_partition = input_size_per_partition
154
+ layer.output_size_per_partition = output_size_per_partition
155
+ layer.orig_dtype = params_dtype
156
+
157
# WEIGHT
158
weight_dtype = (torch.float8_e4m3fn
159
if self.quant_config.is_checkpoint_fp8_serialized else
@@ -172,6 +187,65 @@ class Fp8LinearMethod(LinearMethodBase):
187
output_partition_sizes=output_partition_sizes,
188
**extra_weight_attrs)
189
190
+ # For GPUs without FP8 hardware support, we use Marlin for fast
191
+ # fused dequantization
192
+ if self.use_marlin:
193
+ layer.marlin_state = GPTQMarlinState.REPACK
194
+
195
+ def prepare_layer_for_marlin(self, layer: Module) -> None:
196
+ print_warning_once(
197
+ "Your GPU does not have native support for FP8 computation but "
198
+ "FP8 quantization is being used. Weight-only FP8 compression will "
199
+ "be used leveraging the Marlin kernel. This may degrade "
200
+ "performance for compute-heavy workloads.")
201
+
202
+ part_size_n = layer.output_size_per_partition
203
+ part_size_k = layer.input_size_per_partition
204
+
205
+ assert layer.marlin_state == GPTQMarlinState.REPACK
206
+ layer.marlin_state = GPTQMarlinState.READY
207
+
208
+ device = layer.weight.device
209
+
210
+ # WEIGHTS
211
+ # Repack weights to gptq format (packed int32 elements)
212
+ packed_gptq_qweight = pack_fp8_to_int32(layer.weight)
213
+
214
+ # Repack weights to marlin format
215
+ marlin_qweight = ops.gptq_marlin_repack(
216
+ b_q_weight=packed_gptq_qweight,
217
+ perm=torch.empty(0, dtype=torch.int, device=device),
218
+ size_k=part_size_k,
219
+ size_n=part_size_n,
220
+ num_bits=8,
221
+ )
222
+ layer.weight = Parameter(marlin_qweight, requires_grad=False)
223
+
224
+ # WEIGHT SCALES
225
+ # Currently Marlin doesn't support per-tensor scales, so we
226
+ # expand it to channelwise
227
+ scales = layer.weight_scale.repeat(1, part_size_n).to(
228
+ layer.orig_dtype).to(device)
229
+ # Permute scales
230
+ marlin_scales = marlin_permute_scales(
231
+ s=scales,
232
+ size_k=part_size_k,
233
+ size_n=part_size_n,
234
+ group_size=-1,
235
+ num_bits=8,
236
+ )
237
+ layer.weight_scale = Parameter(marlin_scales, requires_grad=False)
238
+
239
+ # Allocate marlin workspace
240
+ max_workspace_size = (
241
+ part_size_n // GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
242
+ workspace = torch.zeros(max_workspace_size,
243
+ dtype=torch.int,
244
+ device=device,
245
+ requires_grad=False)
246
+
247
+ layer.workspace = workspace
248
+
249
def process_weights_after_loading(self, layer: Module) -> None:
250
if (not hasattr(layer, "process_after_load")
251
or not layer.process_after_load):
@@ -185,6 +259,8 @@ class Fp8LinearMethod(LinearMethodBase):
259
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
260
layer.logical_widths = None
261
layer.input_scale = None
262
+ if self.use_marlin:
263
+ self.prepare_layer_for_marlin(layer)
264
return
265
266
# If checkpoint is fp8, requantize the separately quantized logical
@@ -233,44 +309,72 @@ class Fp8LinearMethod(LinearMethodBase):
309
raise ValueError(
310
f"Unknown scheme {self.quant_config.activation_scheme}")
311
312
+ if self.use_marlin:
313
+ self.prepare_layer_for_marlin(layer)
314
+
315
def apply(self,
316
layer: torch.nn.Module,
317
x: torch.Tensor,
318
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
319
241
- # ops.scaled_fp8_quant supports both dynamic and static quant.
242
- # If dynamic, layer.input_scale is None and x_scale computed from x.
243
- # If static, layer.input_scale is scalar and x_scale is input_scale.
320
+ if self.use_marlin:
321
+ # For GPUs that lack FP8 hardware support, we can leverage the
322
+ # Marlin kernel for fast weight-only FP8 quantization
323
+
324
+ reshaped_x = x.reshape(-1, x.shape[-1])
325
+ out_shape = x.shape[:-1] + (layer.output_size_per_partition, )
326
+
327
+ output = ops.fp8_marlin_gemm(
328
+ a=reshaped_x,
329
+ b_q_weight=layer.weight,
330
+ b_scales=layer.weight_scale,
331
+ workspace=layer.workspace,
332
+ num_bits=8,
333
+ size_m=reshaped_x.shape[0],
334
+ size_n=layer.output_size_per_partition,
335
+ size_k=layer.input_size_per_partition,
336
+ )
337
245
- if bias is None and self.cutlass_fp8_supported:
246
- qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
338
+ if bias is not None:
339
+ output.add_(bias) # In-place add
340
248
- # Fused GEMM_DQ
249
- output = ops.cutlass_scaled_mm(
250
- qinput,
251
- layer.weight,
252
- out_dtype=x.dtype,
253
- scale_a=x_scale,
254
- scale_b=layer.weight_scale,
255
- )
341
+ return output.reshape(out_shape)
342
343
else:
258
- qinput, x_scale = ops.scaled_fp8_quant(x,
259
- layer.input_scale,
260
- batch_dim_padding=17)
261
-
262
- # Fused GEMM_DQ -- note we padded the input above because
263
- # torch._scaled_mm is more performant for matrices with
264
- # batch dimension > 16. Note that this could change
265
- # in the future.
266
- output, _ = torch._scaled_mm(
267
- qinput,
268
- layer.weight,
269
- out_dtype=x.dtype,
270
- scale_a=x_scale,
271
- scale_b=layer.weight_scale,
272
- bias=bias,
273
- )
344
+
345
+ # ops.scaled_fp8_quant supports both dynamic and static quant.
346
+ # If dynamic, layer.input_scale is None and x_scale computed from x
347
+ # If static, layer.input_scale is scalar and x_scale is input_scale
348
+
349
+ if bias is None and self.cutlass_fp8_supported:
350
+ qinput, x_scale = ops.scaled_fp8_quant(x, layer.input_scale)
351
+
352
+ # Fused GEMM_DQ
353
+ output = ops.cutlass_scaled_mm(
354
+ qinput,
355
+ layer.weight,
356
+ out_dtype=x.dtype,
357
+ scale_a=x_scale,
358
+ scale_b=layer.weight_scale,
359
+ )
360
+
361
+ else:
362
+ qinput, x_scale = ops.scaled_fp8_quant(x,
363
+ layer.input_scale,
364
+ batch_dim_padding=17)
365
+
366
+ # Fused GEMM_DQ -- note we padded the input above because
367
+ # torch._scaled_mm is more performant for matrices with
368
+ # batch dimension > 16. Note that this could change
369
+ # in the future.
370
+ output, _ = torch._scaled_mm(
371
+ qinput,
372
+ layer.weight,
373
+ out_dtype=x.dtype,
374
+ scale_a=x_scale,
375
+ scale_b=layer.weight_scale,
376
+ bias=bias,
377
+ )
378
379
return torch.narrow(output, 0, 0, x.shape[0])
380
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+25
-3
@@ -14,13 +14,12 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
14
get_pack_factor, quantize_weights, sort_weights)
15
from vllm.platforms import current_platform
16
17
-__cuda_arch = current_platform.get_device_capability()
18
-
17
MARLIN_TILE = 16
18
19
20
def is_marlin_supported():
23
- return __cuda_arch[0] >= 8
21
+ capability = current_platform.get_device_capability()
22
+ return capability[0] >= 8
23
24
25
def marlin_permute_weights(q_w, size_k, size_n, perm, tile=MARLIN_TILE):
@@ -223,3 +222,26 @@ class MarlinWorkspace:
222
self.scratch = torch.zeros(max_workspace_size,
223
dtype=torch.int,
224
device="cuda")
225
+
226
+
227
+def pack_fp8_to_int32(fp8_tensor: torch.Tensor) -> torch.Tensor:
228
+ """
229
+ Repack FP8 weights to gptq format (packed int32 elements)
230
+ """
231
+ assert fp8_tensor.dtype == torch.float8_e4m3fn
232
+ assert fp8_tensor.shape[0] % 4 == 0
233
+
234
+ # Reshape to prepare for packing
235
+ reshaped = fp8_tensor.reshape(-1, 4, *fp8_tensor.shape[1:])
236
+
237
+ # Convert fp8 to uint8 (byte) representation
238
+ byte_tensor = reshaped.view(torch.uint8)
239
+
240
+ # Pack 4 uint8 values into one int32
241
+ packed = (byte_tensor[:, 0].to(torch.int32) |
242
+ (byte_tensor[:, 1].to(torch.int32) << 8) |
243
+ (byte_tensor[:, 2].to(torch.int32) << 16) |
244
+ (byte_tensor[:, 3].to(torch.int32) << 24))
245
+
246
+ return packed.view(fp8_tensor.shape[0] // 4,
247
+ *fp8_tensor.shape[1:]).contiguous()