@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4fc5c23bb6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[NVIDIA] Support nvfp4 quantization (#12784)

Kaixi Hou committed Feb 12, 2025 at 19:51 UTC 4fc5c23bb64719bda4f2b24e0b95637ccd530fab
12 files changed +688 -13
CMakeLists.txt
+18
@@ -264,6 +264,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
264 "csrc/custom_all_reduce.cu"
265 "csrc/permute_cols.cu"
266 "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
267 + "csrc/quantization/fp4/nvfp4_quant_entry.cu"
268 "csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
269 "csrc/sparse/cutlass/sparse_compressor_entry.cu"
270 "csrc/cutlass_extensions/common.cpp")
@@ -377,6 +378,23 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
378 endif()
379 endif()
380
381 + # FP4 Archs and flags
382 + cuda_archs_loose_intersection(FP4_ARCHS "10.0a" "${CUDA_ARCHS}")
383 + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.8 AND FP4_ARCHS)
384 + set(SRCS
385 + "csrc/quantization/fp4/nvfp4_quant_kernels.cu"
386 + )
387 + set_gencode_flags_for_srcs(
388 + SRCS "${SRCS}"
389 + CUDA_ARCHS "${FP4_ARCHS}")
390 + list(APPEND VLLM_EXT_SRC "${SRCS}")
391 + list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4=1")
392 + message(STATUS "Building NVFP4 for archs: ${FP4_ARCHS}")
393 + else()
394 + message(STATUS "Not building NVFP4 as no compatible archs were found.")
395 + # clear FP4_ARCHS
396 + set(FP4_ARCHS)
397 + endif()
398
399 #
400 # Machete kernels
cmake/utils.cmake
+13 -5
@@ -257,9 +257,9 @@ endmacro()
257 # where `<=` is the version comparison operator.
258 # In other words, for each version in `TGT_CUDA_ARCHS` find the highest version
259 # in `SRC_CUDA_ARCHS` that is less or equal to the version in `TGT_CUDA_ARCHS`.
260 -# We have special handling for 9.0a, if 9.0a is in `SRC_CUDA_ARCHS` and 9.0 is
261 -# in `TGT_CUDA_ARCHS` then we should remove 9.0a from `SRC_CUDA_ARCHS` and add
262 -# 9.0a to the result (and remove 9.0 from TGT_CUDA_ARCHS).
260 +# We have special handling for x.0a, if x.0a is in `SRC_CUDA_ARCHS` and x.0 is
261 +# in `TGT_CUDA_ARCHS` then we should remove x.0a from `SRC_CUDA_ARCHS` and add
262 +# x.0a to the result (and remove x.0 from TGT_CUDA_ARCHS).
263 # The result is stored in `OUT_CUDA_ARCHS`.
264 #
265 # Example:
@@ -272,8 +272,8 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
272 list(REMOVE_DUPLICATES SRC_CUDA_ARCHS)
273 set(TGT_CUDA_ARCHS_ ${TGT_CUDA_ARCHS})
274
275 - # if 9.0a is in SRC_CUDA_ARCHS and 9.0 is in CUDA_ARCHS then we should
276 - # remove 9.0a from SRC_CUDA_ARCHS and add 9.0a to _CUDA_ARCHS
275 + # if x.0a is in SRC_CUDA_ARCHS and x.0 is in CUDA_ARCHS then we should
276 + # remove x.0a from SRC_CUDA_ARCHS and add x.0a to _CUDA_ARCHS
277 set(_CUDA_ARCHS)
278 if ("9.0a" IN_LIST SRC_CUDA_ARCHS)
279 list(REMOVE_ITEM SRC_CUDA_ARCHS "9.0a")
@@ -283,6 +283,14 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
283 endif()
284 endif()
285
286 + if ("10.0a" IN_LIST SRC_CUDA_ARCHS)
287 + list(REMOVE_ITEM SRC_CUDA_ARCHS "10.0a")
288 + if ("10.0" IN_LIST TGT_CUDA_ARCHS)
289 + list(REMOVE_ITEM TGT_CUDA_ARCHS_ "10.0")
290 + set(_CUDA_ARCHS "10.0a")
291 + endif()
292 + endif()
293 +
294 list(SORT SRC_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
295
296 # for each ARCH in TGT_CUDA_ARCHS find the highest arch in SRC_CUDA_ARCHS that
csrc/cuda_utils.h
+12
@@ -1,5 +1,7 @@
1 #pragma once
2
3 +#include <stdio.h>
4 +
5 #if defined(__CUDACC__) || defined(_NVHPC_CUDA)
6 #define HOST_DEVICE_INLINE __forceinline__ __host__ __device__
7 #define DEVICE_INLINE __forceinline__ __device__
@@ -10,6 +12,16 @@
12 #define HOST_INLINE inline
13 #endif
14
15 +#define CUDA_CHECK(cmd) \
16 + do { \
17 + cudaError_t e = cmd; \
18 + if (e != cudaSuccess) { \
19 + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \
20 + cudaGetErrorString(e)); \
21 + exit(EXIT_FAILURE); \
22 + } \
23 + } while (0)
24 +
25 int64_t get_device_attribute(int64_t attribute, int64_t device_id);
26
27 int64_t get_max_shared_memory_per_block_device_attribute(int64_t device_id);
csrc/cuda_utils_kernels.cu
+14 -8
@@ -1,16 +1,22 @@
1 +#include "cuda_utils.h"
2 #ifdef USE_ROCM
3 #include <hip/hip_runtime.h>
4 #include <hip/hip_runtime_api.h>
5 #endif
6 +
7 int64_t get_device_attribute(int64_t attribute, int64_t device_id) {
6 - int device, value;
7 - if (device_id < 0) {
8 - cudaGetDevice(&device);
9 - } else {
10 - device = device_id;
11 - }
12 - cudaDeviceGetAttribute(&value, static_cast<cudaDeviceAttr>(attribute),
13 - device);
8 + // Return the cached value on subsequent calls
9 + static int value = [=]() {
10 + int device = static_cast<int>(device_id);
11 + if (device < 0) {
12 + CUDA_CHECK(cudaGetDevice(&device));
13 + }
14 + int value;
15 + CUDA_CHECK(cudaDeviceGetAttribute(
16 + &value, static_cast<cudaDeviceAttr>(attribute), device));
17 + return static_cast<int>(value);
18 + }();
19 +
20 return value;
21 }
22
csrc/ops.h
+4
@@ -195,6 +195,10 @@ torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
195
196 void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
197
198 +void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
199 + torch::Tensor& output_scale,
200 + torch::Tensor const& input_scale);
201 +
202 void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
203 torch::Tensor const& scale);
204
csrc/quantization/fp4/nvfp4_quant_entry.cu new
+32
@@ -0,0 +1,32 @@
1 +/*
2 + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +
17 +#include <torch/all.h>
18 +
19 +#if defined ENABLE_NVFP4 && ENABLE_NVFP4
20 +void scaled_fp4_quant_sm100a(torch::Tensor const& output,
21 + torch::Tensor const& input,
22 + torch::Tensor const& output_sf,
23 + torch::Tensor const& input_sf);
24 +#endif
25 +
26 +void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
27 + torch::Tensor& output_sf, torch::Tensor const& input_sf) {
28 +#if defined ENABLE_NVFP4 && ENABLE_NVFP4
29 + return scaled_fp4_quant_sm100a(output, input, output_sf, input_sf);
30 +#endif
31 + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 quantization");
32 +}
csrc/quantization/fp4/nvfp4_quant_kernels.cu new
+379
@@ -0,0 +1,379 @@
1 +/*
2 + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +
17 +#include <torch/all.h>
18 +
19 +#include <cuda_runtime_api.h>
20 +#include <cuda_runtime.h>
21 +
22 +#include <ATen/cuda/CUDAContext.h>
23 +#include <c10/cuda/CUDAGuard.h>
24 +
25 +#include <cuda_fp8.h>
26 +
27 +#include "cuda_utils.h"
28 +
29 +// Get type2 from type or vice versa (applied to half and bfloat16)
30 +template <typename T>
31 +struct TypeConverter {
32 + using Type = half2;
33 +}; // keep for generality
34 +
35 +template <>
36 +struct TypeConverter<half2> {
37 + using Type = half;
38 +};
39 +
40 +template <>
41 +struct TypeConverter<half> {
42 + using Type = half2;
43 +};
44 +
45 +template <>
46 +struct TypeConverter<__nv_bfloat162> {
47 + using Type = __nv_bfloat16;
48 +};
49 +
50 +template <>
51 +struct TypeConverter<__nv_bfloat16> {
52 + using Type = __nv_bfloat162;
53 +};
54 +
55 +#define ELTS_PER_THREAD 8
56 +
57 +constexpr int CVT_FP4_ELTS_PER_THREAD = 8;
58 +constexpr int CVT_FP4_SF_VEC_SIZE = 16;
59 +
60 +// Convert 8 float32 values into 8 e2m1 values (represented as one uint32_t).
61 +inline __device__ uint32_t fp32_vec_to_e2m1(float (&array)[8]) {
62 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
63 + uint32_t val;
64 + asm volatile(
65 + "{\n"
66 + ".reg .b8 byte0;\n"
67 + ".reg .b8 byte1;\n"
68 + ".reg .b8 byte2;\n"
69 + ".reg .b8 byte3;\n"
70 + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n"
71 + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n"
72 + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n"
73 + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n"
74 + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n"
75 + "}"
76 + : "=r"(val)
77 + : "f"(array[0]), "f"(array[1]), "f"(array[2]), "f"(array[3]),
78 + "f"(array[4]), "f"(array[5]), "f"(array[6]), "f"(array[7]));
79 + return val;
80 +#else
81 + return 0;
82 +#endif
83 +}
84 +
85 +// Convert 4 float2 values into 8 e2m1 values (represented as one uint32_t).
86 +inline __device__ uint32_t fp32_vec_to_e2m1(float2 (&array)[4]) {
87 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
88 + uint32_t val;
89 + asm volatile(
90 + "{\n"
91 + ".reg .b8 byte0;\n"
92 + ".reg .b8 byte1;\n"
93 + ".reg .b8 byte2;\n"
94 + ".reg .b8 byte3;\n"
95 + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n"
96 + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n"
97 + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n"
98 + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n"
99 + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n"
100 + "}"
101 + : "=r"(val)
102 + : "f"(array[0].x), "f"(array[0].y), "f"(array[1].x), "f"(array[1].y),
103 + "f"(array[2].x), "f"(array[2].y), "f"(array[3].x), "f"(array[3].y));
104 + return val;
105 +#else
106 + return 0;
107 +#endif
108 +}
109 +
110 +// Fast reciprocal.
111 +inline __device__ float reciprocal_approximate_ftz(float a) {
112 + float b;
113 + asm volatile("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(b) : "f"(a));
114 + return b;
115 +}
116 +
117 +template <class SFType, int CVT_FP4_NUM_THREADS_PER_SF>
118 +__device__ uint8_t* cvt_quant_to_fp4_get_sf_out_offset(int rowIdx, int colIdx,
119 + int numCols,
120 + SFType* SFout) {
121 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
122 + static_assert(CVT_FP4_NUM_THREADS_PER_SF == 1 ||
123 + CVT_FP4_NUM_THREADS_PER_SF == 2);
124 +
125 + // One pair of threads write one SF to global memory.
126 + // TODO: stage through smem for packed STG.32
127 + // is it better than STG.8 from 4 threads ?
128 + if (threadIdx.x % CVT_FP4_NUM_THREADS_PER_SF == 0) {
129 + // SF vector index (16 elements share one SF in the K dimension).
130 + int32_t kIdx = colIdx / CVT_FP4_NUM_THREADS_PER_SF;
131 + int32_t mIdx = rowIdx;
132 +
133 + // SF layout [numMTiles, numKTiles, 32 (mTile), 4 (mTile), 4(kTile)]
134 + // --> index [mTileIdx, kTileIdx, outerMIdx, innerMIdx, innerKIdx]
135 +
136 + int32_t mTileIdx = mIdx / (32 * 4);
137 + // SF vector size 16.
138 + int factor = CVT_FP4_SF_VEC_SIZE * 4;
139 + int32_t numKTiles = (numCols + factor - 1) / factor;
140 + int64_t mTileStride = numKTiles * 32 * 4 * 4;
141 +
142 + int32_t kTileIdx = (kIdx / 4);
143 + int64_t kTileStride = 32 * 4 * 4;
144 +
145 + // M tile layout [32, 4] is column-major.
146 + int32_t outerMIdx = (mIdx % 32);
147 + int64_t outerMStride = 4 * 4;
148 +
149 + int32_t innerMIdx = (mIdx % (32 * 4)) / 32;
150 + int64_t innerMStride = 4;
151 +
152 + int32_t innerKIdx = (kIdx % 4);
153 + int64_t innerKStride = 1;
154 +
155 + // Compute the global offset.
156 + int64_t SFOffset = mTileIdx * mTileStride + kTileIdx * kTileStride +
157 + outerMIdx * outerMStride + innerMIdx * innerMStride +
158 + innerKIdx * innerKStride;
159 +
160 + return reinterpret_cast<uint8_t*>(SFout) + SFOffset;
161 + }
162 +#endif
163 + return nullptr;
164 +}
165 +
166 +// Define a 16 bytes packed data type.
167 +template <class Type>
168 +struct PackedVec {
169 + typename TypeConverter<Type>::Type elts[4];
170 +};
171 +
172 +template <>
173 +struct PackedVec<__nv_fp8_e4m3> {
174 + __nv_fp8x2_e4m3 elts[8];
175 +};
176 +
177 +// Quantizes the provided PackedVec into the uint32_t output
178 +template <class Type, bool UE8M0_SF = false>
179 +__device__ uint32_t cvt_warp_fp16_to_fp4(PackedVec<Type>& vec, float SFScaleVal,
180 + uint8_t* SFout) {
181 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
182 + // Get absolute maximum values among the local 8 values.
183 + auto localMax = __habs2(vec.elts[0]);
184 +
185 + // Local maximum value.
186 + #pragma unroll
187 + for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) {
188 + localMax = __hmax2(localMax, __habs2(vec.elts[i]));
189 + }
190 +
191 + // Get the absolute maximum among all 16 values (two threads).
192 + localMax = __hmax2(__shfl_xor_sync(uint32_t(-1), localMax, 1), localMax);
193 + // Get the final absolute maximum values.
194 + float vecMax = float(__hmax(localMax.x, localMax.y));
195 +
196 + // Get the SF (max value of the vector / max value of e2m1).
197 + // maximum value of e2m1 = 6.0.
198 + // TODO: use half as compute data type.
199 + float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f));
200 + // 8 bits representation of the SF.
201 + uint8_t fp8SFVal;
202 + // Write the SF to global memory (STG.8).
203 + if constexpr (UE8M0_SF) {
204 + // Extract the 8 exponent bits from float32.
205 + // float 32bits = 1 sign bit + 8 exponent bits + 23 mantissa bits.
206 + uint32_t tmp = reinterpret_cast<uint32_t&>(SFValue) >> 23;
207 + fp8SFVal = tmp & 0xff;
208 + // Convert back to fp32.
209 + reinterpret_cast<uint32_t&>(SFValue) = tmp << 23;
210 + } else {
211 + // Here SFValue is always positive, so E4M3 is the same as UE4M3.
212 + __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue);
213 + reinterpret_cast<__nv_fp8_e4m3&>(fp8SFVal) = tmp;
214 + // Convert back to fp32.
215 + SFValue = float(tmp);
216 + }
217 + // Get the output scale.
218 + // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) *
219 + // reciprocal(SFScaleVal))
220 + float outputScale =
221 + SFValue != 0 ? reciprocal_approximate_ftz(
222 + SFValue * reciprocal_approximate_ftz(SFScaleVal))
223 + : 0.0f;
224 +
225 + if (SFout) {
226 + // Write the SF to global memory (STG.8).
227 + *SFout = fp8SFVal;
228 + }
229 +
230 + // Convert the input to float.
231 + float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2];
232 +
233 + #pragma unroll
234 + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) {
235 + if constexpr (std::is_same_v<Type, half>) {
236 + fp2Vals[i] = __half22float2(vec.elts[i]);
237 + } else {
238 + fp2Vals[i] = __bfloat1622float2(vec.elts[i]);
239 + }
240 + fp2Vals[i].x *= outputScale;
241 + fp2Vals[i].y *= outputScale;
242 + }
243 +
244 + // Convert to e2m1 values.
245 + uint32_t e2m1Vec = fp32_vec_to_e2m1(fp2Vals);
246 +
247 + // Write the e2m1 values to global memory.
248 + return e2m1Vec;
249 +#else
250 + return 0;
251 +#endif
252 +}
253 +
254 +// Use UE4M3 by default.
255 +template <class Type, bool UE8M0_SF = false>
256 +__global__ void
257 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
258 +__launch_bounds__(512, 4) cvt_fp16_to_fp4(
259 +#else
260 +cvt_fp16_to_fp4(
261 +#endif
262 + int32_t numRows, int32_t numCols, Type const* in, float const* SFScale,
263 + uint32_t* out, uint32_t* SFout) {
264 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000)
265 + using PackedVec = PackedVec<Type>;
266 + static constexpr int CVT_FP4_NUM_THREADS_PER_SF =
267 + (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD);
268 + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
269 + "Vec size is not matched.");
270 +
271 + // Get the global scaling factor, which will be applied to the SF.
272 + // Note SFScale is the same as next GEMM's alpha, which is
273 + // (448.f / (Alpha_A / 6.f)).
274 + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[0];
275 +
276 + // Input tensor row/col loops.
277 + for (int rowIdx = blockIdx.x; rowIdx < numRows; rowIdx += gridDim.x) {
278 + for (int colIdx = threadIdx.x; colIdx < numCols / CVT_FP4_ELTS_PER_THREAD;
279 + colIdx += blockDim.x) {
280 + int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
281 + PackedVec in_vec = reinterpret_cast<PackedVec const*>(in)[inOffset];
282 + // Get the output tensor offset.
283 + // Same as inOffset because 8 elements are packed into one uint32_t.
284 + int64_t outOffset = inOffset;
285 + auto& out_pos = out[outOffset];
286 +
287 + auto sf_out =
288 + cvt_quant_to_fp4_get_sf_out_offset<uint32_t,
289 + CVT_FP4_NUM_THREADS_PER_SF>(
290 + rowIdx, colIdx, numCols, SFout);
291 +
292 + out_pos =
293 + cvt_warp_fp16_to_fp4<Type, UE8M0_SF>(in_vec, SFScaleVal, sf_out);
294 + }
295 + }
296 +#endif
297 +}
298 +
299 +template <typename T>
300 +void invokeFP4Quantization(int m, int n, T const* input, float const* SFScale,
301 + int64_t* output, int32_t* SFOuput, bool useUE8M0,
302 + int multiProcessorCount, cudaStream_t stream) {
303 + // Grid, Block size.
304 + // Each thread converts 8 values.
305 + dim3 block(std::min(int(n / ELTS_PER_THREAD), 512));
306 + // Get number of blocks per SM (assume we can fully utilize the SM).
307 + int const numBlocksPerSM = 2048 / block.x;
308 + dim3 grid(std::min(int(m), multiProcessorCount * numBlocksPerSM));
309 +
310 + // Launch the cvt kernel.
311 + if (useUE8M0) {
312 + cvt_fp16_to_fp4<T, true><<<grid, block, 0, stream>>>(
313 + m, n, input, SFScale, reinterpret_cast<uint32_t*>(output),
314 + reinterpret_cast<uint32_t*>(SFOuput));
315 + } else {
316 + cvt_fp16_to_fp4<T, false><<<grid, block, 0, stream>>>(
317 + m, n, input, SFScale, reinterpret_cast<uint32_t*>(output),
318 + reinterpret_cast<uint32_t*>(SFOuput));
319 + }
320 +}
321 +
322 +// Instantiate the function.
323 +template void invokeFP4Quantization(int m, int n, half const* input,
324 + float const* SFScale, int64_t* output,
325 + int32_t* SFOuput, bool useUE8M0,
326 + int multiProcessorCount,
327 + cudaStream_t stream);
328 +
329 +template void invokeFP4Quantization(int m, int n, __nv_bfloat16 const* input,
330 + float const* SFScale, int64_t* output,
331 + int32_t* SFOuput, bool useUE8M0,
332 + int multiProcessorCount,
333 + cudaStream_t stream);
334 +
335 +void scaled_fp4_quant_sm100a(torch::Tensor const& output,
336 + torch::Tensor const& input,
337 + torch::Tensor const& output_sf,
338 + torch::Tensor const& input_sf) {
339 + int32_t m = input.size(0);
340 + int32_t n = input.size(1);
341 +
342 + TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16.");
343 +
344 + int multiProcessorCount =
345 + get_device_attribute(cudaDevAttrMultiProcessorCount, -1);
346 +
347 + auto input_sf_ptr = static_cast<float const*>(input_sf.data_ptr());
348 + auto sf_out = static_cast<int32_t*>(output_sf.data_ptr());
349 + auto output_ptr = static_cast<int64_t*>(output.data_ptr());
350 + at::cuda::CUDAGuard device_guard{(char)input.get_device()};
351 + auto stream = at::cuda::getStreamFromPool(false, input.get_device());
352 + if (stream == nullptr) {
353 + std::cerr << "Warning: Null CUDA stream" << std::endl;
354 + }
355 +
356 + // We don't support e8m0 scales at this moment.
357 + bool useUE8M0 = false;
358 +
359 + switch (input.scalar_type()) {
360 + case torch::kHalf: {
361 + auto input_ptr = reinterpret_cast<half const*>(input.data_ptr());
362 + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out,
363 + useUE8M0, multiProcessorCount, stream);
364 + break;
365 + }
366 + case torch::kBFloat16: {
367 + auto input_ptr = reinterpret_cast<__nv_bfloat16 const*>(input.data_ptr());
368 + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out,
369 + useUE8M0, multiProcessorCount, stream);
370 + break;
371 + }
372 + default: {
373 + std::cerr << "Observing: " << input.scalar_type()
374 + << " for the input datatype which is invalid";
375 + throw std::runtime_error(
376 + "Unsupported input data type for quantize_to_fp4.");
377 + }
378 + }
379 +}
csrc/torch_bindings.cpp
+6
@@ -423,6 +423,12 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
423 ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
424 &dynamic_per_token_scaled_fp8_quant);
425
426 + // Compute NVFP4 block quantized tensor.
427 + ops.def(
428 + "scaled_fp4_quant(Tensor! output, Tensor input,"
429 + " Tensor! output_scale, Tensor input_scale) -> ()");
430 + ops.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant);
431 +
432 // Compute int8 quantized tensor for given scaling factor.
433 ops.def(
434 "static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
tests/kernels/test_nvfp4_quant.py new
+149
@@ -0,0 +1,149 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import pytest
3 +import torch
4 +
5 +from vllm import _custom_ops as ops
6 +from vllm.platforms import current_platform
7 +from vllm.scalar_type import scalar_types
8 +
9 +if not current_platform.has_device_capability(100):
10 + pytest.skip(reason="Nvfp4 Requires compute capability of 10 or above.",
11 + allow_module_level=True)
12 +
13 +DTYPES = [torch.float16, torch.bfloat16]
14 +SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)]
15 +PAD_SHAPES = [(90, 64), (150, 64), (128, 48), (128, 80), (150, 80), (90, 48),
16 + (90, 128), (150, 128), (150, 48), (90, 80)]
17 +SEEDS = [42]
18 +CUDA_DEVICES = ['cuda:0']
19 +
20 +FLOAT4_E2M1_MAX = scalar_types.float4_e2m1fn.max()
21 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
22 +
23 +# E2M1 to float
24 +# 0111 -> 6
25 +# 0110 -> 4
26 +# 0101 -> 3
27 +# 0100 -> 2
28 +# 0011 -> 1.5
29 +# 0010 -> 1
30 +# 0001 -> 0.5
31 +# 0000 -> 0
32 +E2M1_TO_FLOAT32 = [
33 + 0., 0.5, 1., 1.5, 2., 3., 4., 6., 0., -0.5, -1., -1.5, -2., -3., -4., -6.
34 +]
35 +BLOCK_SIZE = 16
36 +
37 +
38 +def cast_from_fp4(x, m, n):
39 + # The fp4 values are packed in uint8 as [v_1st | v_2nd]
40 + v_2nd = x & 0xF
41 + v_1st = (x >> 4) & 0xF
42 + c = torch.stack((v_2nd, v_1st), dim=-1)
43 + out = torch.tensor([E2M1_TO_FLOAT32[x] for x in c.flatten()])
44 + out = out.reshape(m, n).to(torch.float32)
45 + return out
46 +
47 +
48 +def cast_to_fp4(x):
49 + sign = torch.sign(x)
50 + x = torch.abs(x)
51 + x[(x >= 0.0) & (x <= 0.25)] = 0.0
52 + x[(x > 0.25) & (x < 0.75)] = 0.5
53 + x[(x >= 0.75) & (x <= 1.25)] = 1.0
54 + x[(x > 1.25) & (x < 1.75)] = 1.5
55 + x[(x >= 1.75) & (x <= 2.5)] = 2.0
56 + x[(x > 2.5) & (x < 3.5)] = 3.0
57 + x[(x >= 3.5) & (x <= 5.0)] = 4.0
58 + x[x > 5.0] = 6.0
59 + return x * sign
60 +
61 +
62 +def get_reciprocal(x):
63 + if isinstance(x, torch.Tensor):
64 + return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x)
65 + elif isinstance(x, (float, int)):
66 + return 0.0 if x == 0 else 1.0 / x
67 + else:
68 + raise TypeError("Input must be a float, int, or a torch.Tensor.")
69 +
70 +
71 +def ref_nvfp4_quant(x, global_scale):
72 + assert global_scale.dtype == torch.float32
73 + assert x.ndim == 2
74 + m, n = x.shape
75 + x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE))
76 + vec_max = torch.max(torch.abs(x), dim=-1,
77 + keepdim=True)[0].to(torch.float32)
78 + scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
79 + scale = scale.to(torch.float8_e4m3fn).to(torch.float32)
80 + output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
81 +
82 + scaled_x = x.to(torch.float32) * output_scale
83 + clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n)
84 + return cast_to_fp4(clipped_x), scale.squeeze(-1)
85 +
86 +
87 +def recover_swizzled_scales(scale, m, n):
88 + round_up = lambda x, y: (x + y - 1) // y * y
89 + rounded_m = round_up(m, 128)
90 + scale_n = n // BLOCK_SIZE
91 + rounded_n = round_up(scale_n, 4)
92 + # Recover the swizzled scaling factor to linear layout
93 + tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4))
94 + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
95 + result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32)
96 + return result[:m, :scale_n]
97 +
98 +
99 +@pytest.mark.parametrize("dtype", DTYPES)
100 +@pytest.mark.parametrize("shape", SHAPES)
101 +@pytest.mark.parametrize("seed", SEEDS)
102 +@pytest.mark.parametrize("device", CUDA_DEVICES)
103 +@torch.inference_mode()
104 +def test_quantize_to_fp4(
105 + dtype: torch.dtype,
106 + shape: tuple[int, int],
107 + seed: int,
108 + device: str,
109 +) -> None:
110 + current_platform.seed_everything(seed)
111 + torch.set_default_device(device)
112 +
113 + m, n = shape
114 +
115 + x = torch.randn((m, n), dtype=dtype)
116 + tensor_amax = torch.abs(x).max().to(torch.float32)
117 + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
118 + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
119 +
120 + out, out_scale = ops.scaled_fp4_quant(x, global_scale)
121 + scale_ans = recover_swizzled_scales(out_scale, m, n)
122 + out_ans = cast_from_fp4(out, m, n)
123 +
124 + torch.testing.assert_close(out_ans, out_ref)
125 + torch.testing.assert_close(scale_ans, scale_ref)
126 +
127 +
128 +@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
129 +@torch.inference_mode()
130 +def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None:
131 + dtype = torch.float16
132 + current_platform.seed_everything(42)
133 + torch.set_default_device('cuda:0')
134 +
135 + m, n = pad_shape
136 +
137 + x = torch.randn((m, n), dtype=dtype)
138 +
139 + tensor_amax = torch.abs(x).max().to(torch.float32)
140 + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
141 + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
142 +
143 + out, out_scale = ops.scaled_fp4_quant(x, global_scale)
144 +
145 + scale_ans = recover_swizzled_scales(out_scale, m, n)
146 + out_ans = cast_from_fp4(out, m, n)
147 +
148 + torch.testing.assert_close(out_ans, out_ref)
149 + torch.testing.assert_close(scale_ans, scale_ref)
tests/test_scalartype.py
+1
@@ -11,6 +11,7 @@ from vllm.scalar_type import scalar_types
11 (0, 15, scalar_types.uint4),
12 (-8, 7, scalar_types.uint4b8),
13 (-128, 127, scalar_types.uint8b128),
14 + (-6., 6., scalar_types.float4_e2m1fn),
15 (-28., 28., scalar_types.float6_e3m2f),
16 (torch.int8, scalar_types.int8),
17 (torch.uint8, scalar_types.uint8),
vllm/_custom_ops.py
+57
@@ -765,6 +765,63 @@ def permute_cols(a: torch.Tensor, perm: torch.Tensor) -> torch.Tensor:
765 return torch.ops._C.permute_cols(a, perm)
766
767
768 +# fp4
769 +def scaled_fp4_quant(
770 + input: torch.Tensor,
771 + input_global_scale: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
772 + """
773 + Quantize input tensor to FP4 and return quantized tensor and scale.
774 +
775 + This function quantizes the last dimension of the given tensor `input`. For
776 + every 16 consecutive elements, a single dynamically computed scaling factor
777 + is shared. This scaling factor is quantized using the `input_global_scale`
778 + and is stored in a swizzled layout (see
779 + https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-b-layout-4x).
780 +
781 + Args:
782 + input: The input tensor to be quantized to FP4
783 + input_global_scale: A scalar scaling factor for the entire tensor.
784 +
785 + Returns:
786 + Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every
787 + two values are packed into a uint8 and float8_e4m3 scaling factors
788 + in the sizzled layout.
789 + """
790 + assert input.ndim >= 1, (
791 + f'input.ndim needs to be >= 1, but got {input.ndim}.')
792 + other_dims = 1 if input.ndim == 1 else -1
793 + input = input.reshape(other_dims, input.shape[-1])
794 + m, n = input.shape
795 + block_size = 16
796 + device = input.device
797 +
798 + assert n % block_size == 0, (
799 + f'last dim has to be multiple of 16, but got {n}.')
800 + assert input.dtype in (torch.float16, torch.bfloat16), (
801 + f'input.dtype needs to be fp16 or bf16 but got {input.dtype}.')
802 +
803 + # Two fp4 values will be packed into an uint8.
804 + output = torch.empty((m, n // 2), device=device, dtype=torch.uint8)
805 +
806 + # We use the rounded values to store the swizzled values. Due to the
807 + # requirement of the Tensor Core, the minimum tile is 128x4 for the scales.
808 + # So, we first pad the scales to multiples of 128 and 4. Then, the scales
809 + # (in float8_e4m3fn) are packed into an int32 for every 4 values. More:
810 + # https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-b-layout-4x
811 + round_up = lambda x, y: (x + y - 1) // y * y
812 + rounded_m = round_up(m, 128)
813 + scale_n = n // block_size
814 + rounded_n = round_up(scale_n, 4)
815 + output_scale = torch.empty((rounded_m, rounded_n // 4),
816 + device=device,
817 + dtype=torch.int32)
818 +
819 + torch.ops._C.scaled_fp4_quant(output, input, output_scale,
820 + input_global_scale)
821 + output_scale = output_scale.view(torch.float8_e4m3fn)
822 + return output, output_scale
823 +
824 +
825 # fp8
826 def scaled_fp8_quant(
827 input: torch.Tensor,
vllm/scalar_type.py
+3
@@ -321,6 +321,9 @@ class scalar_types:
321 # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main
322 float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE)
323
324 + # fp4, https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
325 + float4_e2m1fn = ScalarType.float_(2, 1, True, NanRepr.NONE)
326 +
327 # "gptq" types
328 uint2b2 = ScalarType.uint(2, 2)
329 uint3b4 = ScalarType.uint(3, 4)