A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[NVIDIA] Support nvfp4 quantization (#12784)
Kaixi Hou committed
Feb 12, 2025 at 19:51 UTC
4fc5c23bb64719bda4f2b24e0b95637ccd530fab
12 files changed
+688
-13
CMakeLists.txt
+18
@@ -264,6 +264,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
264
"csrc/custom_all_reduce.cu"
265
"csrc/permute_cols.cu"
266
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
267
+ "csrc/quantization/fp4/nvfp4_quant_entry.cu"
268
"csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
269
"csrc/sparse/cutlass/sparse_compressor_entry.cu"
270
"csrc/cutlass_extensions/common.cpp")
@@ -377,6 +378,23 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
378
endif()
379
endif()
380
381
+ # FP4 Archs and flags
382
+ cuda_archs_loose_intersection(FP4_ARCHS "10.0a" "${CUDA_ARCHS}")
383
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND FP4_ARCHS)
384
+ set(SRCS
385
+ "csrc/quantization/fp4/nvfp4_quant_kernels.cu"
386
+ )
387
+ set_gencode_flags_for_srcs(
388
+ SRCS "${SRCS}"
389
+ CUDA_ARCHS "${FP4_ARCHS}")
390
+ list(APPEND VLLM_EXT_SRC "${SRCS}")
391
+ list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4=1")
392
+ message(STATUS "Building NVFP4 for archs: ${FP4_ARCHS}")
393
+ else()
394
+ message(STATUS "Not building NVFP4 as no compatible archs were found.")
395
+ # clear FP4_ARCHS
396
+ set(FP4_ARCHS)
397
+ endif()
398
399
#
400
# Machete kernels
cmake/utils.cmake
+13
-5
@@ -257,9 +257,9 @@ endmacro()
257
# where `<=` is the version comparison operator.
258
# In other words, for each version in `TGT_CUDA_ARCHS` find the highest version
259
# in `SRC_CUDA_ARCHS` that is less or equal to the version in `TGT_CUDA_ARCHS`.
260
-# We have special handling for 9.0a, if 9.0a is in `SRC_CUDA_ARCHS` and 9.0 is
261
-# in `TGT_CUDA_ARCHS` then we should remove 9.0a from `SRC_CUDA_ARCHS` and add
262
-# 9.0a to the result (and remove 9.0 from TGT_CUDA_ARCHS).
260
+# We have special handling for x.0a, if x.0a is in `SRC_CUDA_ARCHS` and x.0 is
261
+# in `TGT_CUDA_ARCHS` then we should remove x.0a from `SRC_CUDA_ARCHS` and add
262
+# x.0a to the result (and remove x.0 from TGT_CUDA_ARCHS).
263
# The result is stored in `OUT_CUDA_ARCHS`.
264
#
265
# Example:
@@ -272,8 +272,8 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
272
list(REMOVE_DUPLICATES SRC_CUDA_ARCHS)
273
set(TGT_CUDA_ARCHS_ ${TGT_CUDA_ARCHS})
274
275
- # if 9.0a is in SRC_CUDA_ARCHS and 9.0 is in CUDA_ARCHS then we should
276
- # remove 9.0a from SRC_CUDA_ARCHS and add 9.0a to _CUDA_ARCHS
275
+ # if x.0a is in SRC_CUDA_ARCHS and x.0 is in CUDA_ARCHS then we should
276
+ # remove x.0a from SRC_CUDA_ARCHS and add x.0a to _CUDA_ARCHS
277
set(_CUDA_ARCHS)
278
if ("9.0a" IN_LIST SRC_CUDA_ARCHS)
279
list(REMOVE_ITEM SRC_CUDA_ARCHS "9.0a")
@@ -283,6 +283,14 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
283
endif()
284
endif()
285
286
+ if ("10.0a" IN_LIST SRC_CUDA_ARCHS)
287
+ list(REMOVE_ITEM SRC_CUDA_ARCHS "10.0a")
288
+ if ("10.0" IN_LIST TGT_CUDA_ARCHS)
289
+ list(REMOVE_ITEM TGT_CUDA_ARCHS_ "10.0")
290
+ set(_CUDA_ARCHS "10.0a")
291
+ endif()
292
+ endif()
293
+
294
list(SORT SRC_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
295
296
# for each ARCH in TGT_CUDA_ARCHS find the highest arch in SRC_CUDA_ARCHS that
csrc/cuda_utils.h
+12
@@ -1,5 +1,7 @@
1
#pragma once
2
3
+#include <stdio.h>
4
+
5
#if defined(__CUDACC__) || defined(_NVHPC_CUDA)
6
#define HOST_DEVICE_INLINE __forceinline__ __host__ __device__
7
#define DEVICE_INLINE __forceinline__ __device__
@@ -10,6 +12,16 @@
12
#define HOST_INLINE inline
13
#endif
14
15
+#define CUDA_CHECK(cmd) \
16
+ do { \
17
+ cudaError_t e = cmd; \
18
+ if (e != cudaSuccess) { \
19
+ printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \
20
+ cudaGetErrorString(e)); \
21
+ exit(EXIT_FAILURE); \
22
+ } \
23
+ } while (0)
24
+
25
int64_t get_device_attribute(int64_t attribute, int64_t device_id);
26
27
int64_t get_max_shared_memory_per_block_device_attribute(int64_t device_id);
csrc/cuda_utils_kernels.cu
+14
-8
@@ -1,16 +1,22 @@
1
+#include "cuda_utils.h"
2
#ifdef USE_ROCM
3
#include <hip/hip_runtime.h>
4
#include <hip/hip_runtime_api.h>
5
#endif
6
+
7
int64_t get_device_attribute(int64_t attribute, int64_t device_id) {
6
- int device, value;
7
- if (device_id < 0) {
8
- cudaGetDevice(&device);
9
- } else {
10
- device = device_id;
11
- }
12
- cudaDeviceGetAttribute(&value, static_cast<cudaDeviceAttr>(attribute),
13
- device);
8
+ // Return the cached value on subsequent calls
9
+ static int value = [=]() {
10
+ int device = static_cast<int>(device_id);
11
+ if (device < 0) {
12
+ CUDA_CHECK(cudaGetDevice(&device));
13
+ }
14
+ int value;
15
+ CUDA_CHECK(cudaDeviceGetAttribute(
16
+ &value, static_cast<cudaDeviceAttr>(attribute), device));
17
+ return static_cast<int>(value);
18
+ }();
19
+
20
return value;
21
}
22
csrc/ops.h
+4
@@ -195,6 +195,10 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
195
196
void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
197
198
+void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
199
+ torch::Tensor& output_scale,
200
+ torch::Tensor const& input_scale);
201
+
202
void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
203
torch::Tensor const& scale);
204
csrc/quantization/fp4/nvfp4_quant_entry.cu
new
+32
@@ -0,0 +1,32 @@
1
+/*
2
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+#include <torch/all.h>
18
+
19
+#if defined ENABLE_NVFP4 && ENABLE_NVFP4
20
+void scaled_fp4_quant_sm100a(torch::Tensor const& output,
21
+ torch::Tensor const& input,
22
+ torch::Tensor const& output_sf,
23
+ torch::Tensor const& input_sf);
24
+#endif
25
+
26
+void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
27
+ torch::Tensor& output_sf, torch::Tensor const& input_sf) {
28
+#if defined ENABLE_NVFP4 && ENABLE_NVFP4
29
+ return scaled_fp4_quant_sm100a(output, input, output_sf, input_sf);
30
+#endif
31
+ TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 quantization");
32
+}
csrc/quantization/fp4/nvfp4_quant_kernels.cu
new
+379
@@ -0,0 +1,379 @@
1
+/*
2
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+#include <torch/all.h>
18
+
19
+#include <cuda_runtime_api.h>
20
+#include <cuda_runtime.h>
21
+
22
+#include <ATen/cuda/CUDAContext.h>
23
+#include <c10/cuda/CUDAGuard.h>
24
+
25
+#include <cuda_fp8.h>
26
+
27
+#include "cuda_utils.h"
28
+
29
+// Get type2 from type or vice versa (applied to half and bfloat16)
30
+template <typename T>
31
+struct TypeConverter {
32
+ using Type = half2;
33
+}; // keep for generality
34
+
35
+template <>
36
+struct TypeConverter<half2> {
37
+ using Type = half;
38
+};
39
+
40
+template <>
41
+struct TypeConverter<half> {
42
+ using Type = half2;
43
+};
44
+
45
+template <>
46
+struct TypeConverter<__nv_bfloat162> {
47
+ using Type = __nv_bfloat16;
48
+};
49
+
50
+template <>
51
+struct TypeConverter<__nv_bfloat16> {
52
+ using Type = __nv_bfloat162;
53
+};
54
+
55
+#define ELTS_PER_THREAD 8
56
+
57
+constexpr int CVT_FP4_ELTS_PER_THREAD = 8;
58
+constexpr int CVT_FP4_SF_VEC_SIZE = 16;
59
+
60
+// Convert 8 float32 values into 8 e2m1 values (represented as one uint32_t).
61
+inline __device__ uint32_t fp32_vec_to_e2m1(float (&array)[8]) {
62
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
63
+ uint32_t val;
64
+ asm volatile(
65
+ "{\n"
66
+ ".reg .b8 byte0;\n"
67
+ ".reg .b8 byte1;\n"
68
+ ".reg .b8 byte2;\n"
69
+ ".reg .b8 byte3;\n"
70
+ "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n"
71
+ "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n"
72
+ "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n"
73
+ "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n"
74
+ "mov.b32 %0, {byte0, byte1, byte2, byte3};\n"
75
+ "}"
76
+ : "=r"(val)
77
+ : "f"(array[0]), "f"(array[1]), "f"(array[2]), "f"(array[3]),
78
+ "f"(array[4]), "f"(array[5]), "f"(array[6]), "f"(array[7]));
79
+ return val;
80
+#else
81
+ return 0;
82
+#endif
83
+}
84
+
85
+// Convert 4 float2 values into 8 e2m1 values (represented as one uint32_t).
86
+inline __device__ uint32_t fp32_vec_to_e2m1(float2 (&array)[4]) {
87
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
88
+ uint32_t val;
89
+ asm volatile(
90
+ "{\n"
91
+ ".reg .b8 byte0;\n"
92
+ ".reg .b8 byte1;\n"
93
+ ".reg .b8 byte2;\n"
94
+ ".reg .b8 byte3;\n"
95
+ "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n"
96
+ "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n"
97
+ "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n"
98
+ "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n"
99
+ "mov.b32 %0, {byte0, byte1, byte2, byte3};\n"
100
+ "}"
101
+ : "=r"(val)
102
+ : "f"(array[0].x), "f"(array[0].y), "f"(array[1].x), "f"(array[1].y),
103
+ "f"(array[2].x), "f"(array[2].y), "f"(array[3].x), "f"(array[3].y));
104
+ return val;
105
+#else
106
+ return 0;
107
+#endif
108
+}
109
+
110
+// Fast reciprocal.
111
+inline __device__ float reciprocal_approximate_ftz(float a) {
112
+ float b;
113
+ asm volatile("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(b) : "f"(a));
114
+ return b;
115
+}
116
+
117
+template <class SFType, int CVT_FP4_NUM_THREADS_PER_SF>
118
+__device__ uint8_t* cvt_quant_to_fp4_get_sf_out_offset(int rowIdx, int colIdx,
119
+ int numCols,
120
+ SFType* SFout) {
121
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
122
+ static_assert(CVT_FP4_NUM_THREADS_PER_SF == 1 ||
123
+ CVT_FP4_NUM_THREADS_PER_SF == 2);
124
+
125
+ // One pair of threads write one SF to global memory.
126
+ // TODO: stage through smem for packed STG.32
127
+ // is it better than STG.8 from 4 threads ?
128
+ if (threadIdx.x % CVT_FP4_NUM_THREADS_PER_SF == 0) {
129
+ // SF vector index (16 elements share one SF in the K dimension).
130
+ int32_t kIdx = colIdx / CVT_FP4_NUM_THREADS_PER_SF;
131
+ int32_t mIdx = rowIdx;
132
+
133
+ // SF layout [numMTiles, numKTiles, 32 (mTile), 4 (mTile), 4(kTile)]
134
+ // --> index [mTileIdx, kTileIdx, outerMIdx, innerMIdx, innerKIdx]
135
+
136
+ int32_t mTileIdx = mIdx / (32 * 4);
137
+ // SF vector size 16.
138
+ int factor = CVT_FP4_SF_VEC_SIZE * 4;
139
+ int32_t numKTiles = (numCols + factor - 1) / factor;
140
+ int64_t mTileStride = numKTiles * 32 * 4 * 4;
141
+
142
+ int32_t kTileIdx = (kIdx / 4);
143
+ int64_t kTileStride = 32 * 4 * 4;
144
+
145
+ // M tile layout [32, 4] is column-major.
146
+ int32_t outerMIdx = (mIdx % 32);
147
+ int64_t outerMStride = 4 * 4;
148
+
149
+ int32_t innerMIdx = (mIdx % (32 * 4)) / 32;
150
+ int64_t innerMStride = 4;
151
+
152
+ int32_t innerKIdx = (kIdx % 4);
153
+ int64_t innerKStride = 1;
154
+
155
+ // Compute the global offset.
156
+ int64_t SFOffset = mTileIdx * mTileStride + kTileIdx * kTileStride +
157
+ outerMIdx * outerMStride + innerMIdx * innerMStride +
158
+ innerKIdx * innerKStride;
159
+
160
+ return reinterpret_cast<uint8_t*>(SFout) + SFOffset;
161
+ }
162
+#endif
163
+ return nullptr;
164
+}
165
+
166
+// Define a 16 bytes packed data type.
167
+template <class Type>
168
+struct PackedVec {
169
+ typename TypeConverter<Type>::Type elts[4];
170
+};
171
+
172
+template <>
173
+struct PackedVec<__nv_fp8_e4m3> {
174
+ __nv_fp8x2_e4m3 elts[8];
175
+};
176
+
177
+// Quantizes the provided PackedVec into the uint32_t output
178
+template <class Type, bool UE8M0_SF = false>
179
+__device__ uint32_t cvt_warp_fp16_to_fp4(PackedVec<Type>& vec, float SFScaleVal,
180
+ uint8_t* SFout) {
181
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
182
+ // Get absolute maximum values among the local 8 values.
183
+ auto localMax = __habs2(vec.elts[0]);
184
+
185
+ // Local maximum value.
186
+ #pragma unroll
187
+ for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) {
188
+ localMax = __hmax2(localMax, __habs2(vec.elts[i]));
189
+ }
190
+
191
+ // Get the absolute maximum among all 16 values (two threads).
192
+ localMax = __hmax2(__shfl_xor_sync(uint32_t(-1), localMax, 1), localMax);
193
+ // Get the final absolute maximum values.
194
+ float vecMax = float(__hmax(localMax.x, localMax.y));
195
+
196
+ // Get the SF (max value of the vector / max value of e2m1).
197
+ // maximum value of e2m1 = 6.0.
198
+ // TODO: use half as compute data type.
199
+ float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f));
200
+ // 8 bits representation of the SF.
201
+ uint8_t fp8SFVal;
202
+ // Write the SF to global memory (STG.8).
203
+ if constexpr (UE8M0_SF) {
204
+ // Extract the 8 exponent bits from float32.
205
+ // float 32bits = 1 sign bit + 8 exponent bits + 23 mantissa bits.
206
+ uint32_t tmp = reinterpret_cast<uint32_t&>(SFValue) >> 23;
207
+ fp8SFVal = tmp & 0xff;
208
+ // Convert back to fp32.
209
+ reinterpret_cast<uint32_t&>(SFValue) = tmp << 23;
210
+ } else {
211
+ // Here SFValue is always positive, so E4M3 is the same as UE4M3.
212
+ __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue);
213
+ reinterpret_cast<__nv_fp8_e4m3&>(fp8SFVal) = tmp;
214
+ // Convert back to fp32.
215
+ SFValue = float(tmp);
216
+ }
217
+ // Get the output scale.
218
+ // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) *
219
+ // reciprocal(SFScaleVal))
220
+ float outputScale =
221
+ SFValue != 0 ? reciprocal_approximate_ftz(
222
+ SFValue * reciprocal_approximate_ftz(SFScaleVal))
223
+ : 0.0f;
224
+
225
+ if (SFout) {
226
+ // Write the SF to global memory (STG.8).
227
+ *SFout = fp8SFVal;
228
+ }
229
+
230
+ // Convert the input to float.
231
+ float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2];
232
+
233
+ #pragma unroll
234
+ for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) {
235
+ if constexpr (std::is_same_v<Type, half>) {
236
+ fp2Vals[i] = __half22float2(vec.elts[i]);
237
+ } else {
238
+ fp2Vals[i] = __bfloat1622float2(vec.elts[i]);
239
+ }
240
+ fp2Vals[i].x *= outputScale;
241
+ fp2Vals[i].y *= outputScale;
242
+ }
243
+
244
+ // Convert to e2m1 values.
245
+ uint32_t e2m1Vec = fp32_vec_to_e2m1(fp2Vals);
246
+
247
+ // Write the e2m1 values to global memory.
248
+ return e2m1Vec;
249
+#else
250
+ return 0;
251
+#endif
252
+}
253
+
254
+// Use UE4M3 by default.
255
+template <class Type, bool UE8M0_SF = false>
256
+__global__ void
257
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
258
+__launch_bounds__(512, 4) cvt_fp16_to_fp4(
259
+#else
260
+cvt_fp16_to_fp4(
261
+#endif
262
+ int32_t numRows, int32_t numCols, Type const* in, float const* SFScale,
263
+ uint32_t* out, uint32_t* SFout) {
264
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
265
+ using PackedVec = PackedVec<Type>;
266
+ static constexpr int CVT_FP4_NUM_THREADS_PER_SF =
267
+ (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD);
268
+ static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
269
+ "Vec size is not matched.");
270
+
271
+ // Get the global scaling factor, which will be applied to the SF.
272
+ // Note SFScale is the same as next GEMM's alpha, which is
273
+ // (448.f / (Alpha_A / 6.f)).
274
+ float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[0];
275
+
276
+ // Input tensor row/col loops.
277
+ for (int rowIdx = blockIdx.x; rowIdx < numRows; rowIdx += gridDim.x) {
278
+ for (int colIdx = threadIdx.x; colIdx < numCols / CVT_FP4_ELTS_PER_THREAD;
279
+ colIdx += blockDim.x) {
280
+ int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
281
+ PackedVec in_vec = reinterpret_cast<PackedVec const*>(in)[inOffset];
282
+ // Get the output tensor offset.
283
+ // Same as inOffset because 8 elements are packed into one uint32_t.
284
+ int64_t outOffset = inOffset;
285
+ auto& out_pos = out[outOffset];
286
+
287
+ auto sf_out =
288
+ cvt_quant_to_fp4_get_sf_out_offset<uint32_t,
289
+ CVT_FP4_NUM_THREADS_PER_SF>(
290
+ rowIdx, colIdx, numCols, SFout);
291
+
292
+ out_pos =
293
+ cvt_warp_fp16_to_fp4<Type, UE8M0_SF>(in_vec, SFScaleVal, sf_out);
294
+ }
295
+ }
296
+#endif
297
+}
298
+
299
+template <typename T>
300
+void invokeFP4Quantization(int m, int n, T const* input, float const* SFScale,
301
+ int64_t* output, int32_t* SFOuput, bool useUE8M0,
302
+ int multiProcessorCount, cudaStream_t stream) {
303
+ // Grid, Block size.
304
+ // Each thread converts 8 values.
305
+ dim3 block(std::min(int(n / ELTS_PER_THREAD), 512));
306
+ // Get number of blocks per SM (assume we can fully utilize the SM).
307
+ int const numBlocksPerSM = 2048 / block.x;
308
+ dim3 grid(std::min(int(m), multiProcessorCount * numBlocksPerSM));
309
+
310
+ // Launch the cvt kernel.
311
+ if (useUE8M0) {
312
+ cvt_fp16_to_fp4<T, true><<<grid, block, 0, stream>>>(
313
+ m, n, input, SFScale, reinterpret_cast<uint32_t*>(output),
314
+ reinterpret_cast<uint32_t*>(SFOuput));
315
+ } else {
316
+ cvt_fp16_to_fp4<T, false><<<grid, block, 0, stream>>>(
317
+ m, n, input, SFScale, reinterpret_cast<uint32_t*>(output),
318
+ reinterpret_cast<uint32_t*>(SFOuput));
319
+ }
320
+}
321
+
322
+// Instantiate the function.
323
+template void invokeFP4Quantization(int m, int n, half const* input,
324
+ float const* SFScale, int64_t* output,
325
+ int32_t* SFOuput, bool useUE8M0,
326
+ int multiProcessorCount,
327
+ cudaStream_t stream);
328
+
329
+template void invokeFP4Quantization(int m, int n, __nv_bfloat16 const* input,
330
+ float const* SFScale, int64_t* output,
331
+ int32_t* SFOuput, bool useUE8M0,
332
+ int multiProcessorCount,
333
+ cudaStream_t stream);
334
+
335
+void scaled_fp4_quant_sm100a(torch::Tensor const& output,
336
+ torch::Tensor const& input,
337
+ torch::Tensor const& output_sf,
338
+ torch::Tensor const& input_sf) {
339
+ int32_t m = input.size(0);
340
+ int32_t n = input.size(1);
341
+
342
+ TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16.");
343
+
344
+ int multiProcessorCount =
345
+ get_device_attribute(cudaDevAttrMultiProcessorCount, -1);
346
+
347
+ auto input_sf_ptr = static_cast<float const*>(input_sf.data_ptr());
348
+ auto sf_out = static_cast<int32_t*>(output_sf.data_ptr());
349
+ auto output_ptr = static_cast<int64_t*>(output.data_ptr());
350
+ at::cuda::CUDAGuard device_guard{(char)input.get_device()};
351
+ auto stream = at::cuda::getStreamFromPool(false, input.get_device());
352
+ if (stream == nullptr) {
353
+ std::cerr << "Warning: Null CUDA stream" << std::endl;
354
+ }
355
+
356
+ // We don't support e8m0 scales at this moment.
357
+ bool useUE8M0 = false;
358
+
359
+ switch (input.scalar_type()) {
360
+ case torch::kHalf: {
361
+ auto input_ptr = reinterpret_cast<half const*>(input.data_ptr());
362
+ invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out,
363
+ useUE8M0, multiProcessorCount, stream);
364
+ break;
365
+ }
366
+ case torch::kBFloat16: {
367
+ auto input_ptr = reinterpret_cast<__nv_bfloat16 const*>(input.data_ptr());
368
+ invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out,
369
+ useUE8M0, multiProcessorCount, stream);
370
+ break;
371
+ }
372
+ default: {
373
+ std::cerr << "Observing: " << input.scalar_type()
374
+ << " for the input datatype which is invalid";
375
+ throw std::runtime_error(
376
+ "Unsupported input data type for quantize_to_fp4.");
377
+ }
378
+ }
379
+}
csrc/torch_bindings.cpp
+6
@@ -423,6 +423,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
423
ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
424
&dynamic_per_token_scaled_fp8_quant);
425
426
+ // Compute NVFP4 block quantized tensor.
427
+ ops.def(
428
+ "scaled_fp4_quant(Tensor! output, Tensor input,"
429
+ " Tensor! output_scale, Tensor input_scale) -> ()");
430
+ ops.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant);
431
+
432
// Compute int8 quantized tensor for given scaling factor.
433
ops.def(
434
"static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
tests/kernels/test_nvfp4_quant.py
new
+149
@@ -0,0 +1,149 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import pytest
3
+import torch
4
+
5
+from vllm import _custom_ops as ops
6
+from vllm.platforms import current_platform
7
+from vllm.scalar_type import scalar_types
8
+
9
+if not current_platform.has_device_capability(100):
10
+ pytest.skip(reason="Nvfp4 Requires compute capability of 10 or above.",
11
+ allow_module_level=True)
12
+
13
+DTYPES = [torch.float16, torch.bfloat16]
14
+SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)]
15
+PAD_SHAPES = [(90, 64), (150, 64), (128, 48), (128, 80), (150, 80), (90, 48),
16
+ (90, 128), (150, 128), (150, 48), (90, 80)]
17
+SEEDS = [42]
18
+CUDA_DEVICES = ['cuda:0']
19
+
20
+FLOAT4_E2M1_MAX = scalar_types.float4_e2m1fn.max()
21
+FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
22
+
23
+# E2M1 to float
24
+# 0111 -> 6
25
+# 0110 -> 4
26
+# 0101 -> 3
27
+# 0100 -> 2
28
+# 0011 -> 1.5
29
+# 0010 -> 1
30
+# 0001 -> 0.5
31
+# 0000 -> 0
32
+E2M1_TO_FLOAT32 = [
33
+ 0., 0.5, 1., 1.5, 2., 3., 4., 6., 0., -0.5, -1., -1.5, -2., -3., -4., -6.
34
+]
35
+BLOCK_SIZE = 16
36
+
37
+
38
+def cast_from_fp4(x, m, n):
39
+ # The fp4 values are packed in uint8 as [v_1st | v_2nd]
40
+ v_2nd = x & 0xF
41
+ v_1st = (x >> 4) & 0xF
42
+ c = torch.stack((v_2nd, v_1st), dim=-1)
43
+ out = torch.tensor([E2M1_TO_FLOAT32[x] for x in c.flatten()])
44
+ out = out.reshape(m, n).to(torch.float32)
45
+ return out
46
+
47
+
48
+def cast_to_fp4(x):
49
+ sign = torch.sign(x)
50
+ x = torch.abs(x)
51
+ x[(x >= 0.0) & (x <= 0.25)] = 0.0
52
+ x[(x > 0.25) & (x < 0.75)] = 0.5
53
+ x[(x >= 0.75) & (x <= 1.25)] = 1.0
54
+ x[(x > 1.25) & (x < 1.75)] = 1.5
55
+ x[(x >= 1.75) & (x <= 2.5)] = 2.0
56
+ x[(x > 2.5) & (x < 3.5)] = 3.0
57
+ x[(x >= 3.5) & (x <= 5.0)] = 4.0
58
+ x[x > 5.0] = 6.0
59
+ return x * sign
60
+
61
+
62
+def get_reciprocal(x):
63
+ if isinstance(x, torch.Tensor):
64
+ return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x)
65
+ elif isinstance(x, (float, int)):
66
+ return 0.0 if x == 0 else 1.0 / x
67
+ else:
68
+ raise TypeError("Input must be a float, int, or a torch.Tensor.")
69
+
70
+
71
+def ref_nvfp4_quant(x, global_scale):
72
+ assert global_scale.dtype == torch.float32
73
+ assert x.ndim == 2
74
+ m, n = x.shape
75
+ x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE))
76
+ vec_max = torch.max(torch.abs(x), dim=-1,
77
+ keepdim=True)[0].to(torch.float32)
78
+ scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
79
+ scale = scale.to(torch.float8_e4m3fn).to(torch.float32)
80
+ output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
81
+
82
+ scaled_x = x.to(torch.float32) * output_scale
83
+ clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n)
84
+ return cast_to_fp4(clipped_x), scale.squeeze(-1)
85
+
86
+
87
+def recover_swizzled_scales(scale, m, n):
88
+ round_up = lambda x, y: (x + y - 1) // y * y
89
+ rounded_m = round_up(m, 128)
90
+ scale_n = n // BLOCK_SIZE
91
+ rounded_n = round_up(scale_n, 4)
92
+ # Recover the swizzled scaling factor to linear layout
93
+ tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4))
94
+ tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
95
+ result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32)
96
+ return result[:m, :scale_n]
97
+
98
+
99
+@pytest.mark.parametrize("dtype", DTYPES)
100
+@pytest.mark.parametrize("shape", SHAPES)
101
+@pytest.mark.parametrize("seed", SEEDS)
102
+@pytest.mark.parametrize("device", CUDA_DEVICES)
103
+@torch.inference_mode()
104
+def test_quantize_to_fp4(
105
+ dtype: torch.dtype,
106
+ shape: tuple[int, int],
107
+ seed: int,
108
+ device: str,
109
+) -> None:
110
+ current_platform.seed_everything(seed)
111
+ torch.set_default_device(device)
112
+
113
+ m, n = shape
114
+
115
+ x = torch.randn((m, n), dtype=dtype)
116
+ tensor_amax = torch.abs(x).max().to(torch.float32)
117
+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
118
+ out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
119
+
120
+ out, out_scale = ops.scaled_fp4_quant(x, global_scale)
121
+ scale_ans = recover_swizzled_scales(out_scale, m, n)
122
+ out_ans = cast_from_fp4(out, m, n)
123
+
124
+ torch.testing.assert_close(out_ans, out_ref)
125
+ torch.testing.assert_close(scale_ans, scale_ref)
126
+
127
+
128
+@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
129
+@torch.inference_mode()
130
+def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None:
131
+ dtype = torch.float16
132
+ current_platform.seed_everything(42)
133
+ torch.set_default_device('cuda:0')
134
+
135
+ m, n = pad_shape
136
+
137
+ x = torch.randn((m, n), dtype=dtype)
138
+
139
+ tensor_amax = torch.abs(x).max().to(torch.float32)
140
+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
141
+ out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
142
+
143
+ out, out_scale = ops.scaled_fp4_quant(x, global_scale)
144
+
145
+ scale_ans = recover_swizzled_scales(out_scale, m, n)
146
+ out_ans = cast_from_fp4(out, m, n)
147
+
148
+ torch.testing.assert_close(out_ans, out_ref)
149
+ torch.testing.assert_close(scale_ans, scale_ref)
tests/test_scalartype.py
+1
@@ -11,6 +11,7 @@ from vllm.scalar_type import scalar_types
11
(0, 15, scalar_types.uint4),
12
(-8, 7, scalar_types.uint4b8),
13
(-128, 127, scalar_types.uint8b128),
14
+ (-6., 6., scalar_types.float4_e2m1fn),
15
(-28., 28., scalar_types.float6_e3m2f),
16
(torch.int8, scalar_types.int8),
17
(torch.uint8, scalar_types.uint8),
vllm/_custom_ops.py
+57
@@ -765,6 +765,63 @@ def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor:
765
return torch.ops._C.permute_cols(a, perm)
766
767
768
+# fp4
769
+def scaled_fp4_quant(
770
+ input: torch.Tensor,
771
+ input_global_scale: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
772
+ """
773
+ Quantize input tensor to FP4 and return quantized tensor and scale.
774
+
775
+ This function quantizes the last dimension of the given tensor `input`. For
776
+ every 16 consecutive elements, a single dynamically computed scaling factor
777
+ is shared. This scaling factor is quantized using the `input_global_scale`
778
+ and is stored in a swizzled layout (see
779
+ https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-b-layout-4x).
780
+
781
+ Args:
782
+ input: The input tensor to be quantized to FP4
783
+ input_global_scale: A scalar scaling factor for the entire tensor.
784
+
785
+ Returns:
786
+ Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every
787
+ two values are packed into a uint8 and float8_e4m3 scaling factors
788
+ in the sizzled layout.
789
+ """
790
+ assert input.ndim >= 1, (
791
+ f'input.ndim needs to be >= 1, but got {input.ndim}.')
792
+ other_dims = 1 if input.ndim == 1 else -1
793
+ input = input.reshape(other_dims, input.shape[-1])
794
+ m, n = input.shape
795
+ block_size = 16
796
+ device = input.device
797
+
798
+ assert n % block_size == 0, (
799
+ f'last dim has to be multiple of 16, but got {n}.')
800
+ assert input.dtype in (torch.float16, torch.bfloat16), (
801
+ f'input.dtype needs to be fp16 or bf16 but got {input.dtype}.')
802
+
803
+ # Two fp4 values will be packed into an uint8.
804
+ output = torch.empty((m, n // 2), device=device, dtype=torch.uint8)
805
+
806
+ # We use the rounded values to store the swizzled values. Due to the
807
+ # requirement of the Tensor Core, the minimum tile is 128x4 for the scales.
808
+ # So, we first pad the scales to multiples of 128 and 4. Then, the scales
809
+ # (in float8_e4m3fn) are packed into an int32 for every 4 values. More:
810
+ # https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-b-layout-4x
811
+ round_up = lambda x, y: (x + y - 1) // y * y
812
+ rounded_m = round_up(m, 128)
813
+ scale_n = n // block_size
814
+ rounded_n = round_up(scale_n, 4)
815
+ output_scale = torch.empty((rounded_m, rounded_n // 4),
816
+ device=device,
817
+ dtype=torch.int32)
818
+
819
+ torch.ops._C.scaled_fp4_quant(output, input, output_scale,
820
+ input_global_scale)
821
+ output_scale = output_scale.view(torch.float8_e4m3fn)
822
+ return output, output_scale
823
+
824
+
825
# fp8
826
def scaled_fp8_quant(
827
input: torch.Tensor,
vllm/scalar_type.py
+3
@@ -321,6 +321,9 @@ class scalar_types:
321
# fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main
322
float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE)
323
324
+ # fp4, https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
325
+ float4_e2m1fn = ScalarType.float_(2, 1, True, NanRepr.NONE)
326
+
327
# "gptq" types
328
uint2b2 = ScalarType.uint(2, 2)
329
uint3b4 = ScalarType.uint(3, 4)