A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 17:45 UTC
[BugFix] Fix fp4 quant kernel on CUDA 12.8 (#35210)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Roberto L. Castro committed
Feb 26, 2026 at 03:32 UTC
86c3b5a808506e325fd7e59d86d83170fc98c93c
2 files changed
+11
-7
csrc/quantization/fp4/activation_nvfp4_quant_fusion_kernels.cu
+4
-2
@@ -107,7 +107,9 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
107
(uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo);
108
reinterpret_cast<uint64_t*>(out)[outOffset >> 1] = packed64;
109
} else {
110
- out[inOffset] = out_val;
110
+ int64_t outOffset =
111
+ rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
112
+ out[outOffset] = out_val;
113
}
114
}
115
}
@@ -140,7 +142,7 @@ void silu_and_mul_nvfp4_quant_sm1xxa(torch::Tensor& output, // [..., d]
142
int const numBlocksPerSM =
143
vllm_runtime_blocks_per_sm(static_cast<int>(block.x));
144
143
- int sf_n_unpadded = int(n / CVT_FP4_SF_VEC_SIZE);
145
+ int sf_n_unpadded = int(n / CVT_FP4_ELTS_PER_THREAD);
146
147
int grid_y = vllm::div_round_up(sf_n_unpadded, static_cast<int>(block.x));
148
int grid_x = std::min(
csrc/quantization/fp4/nvfp4_quant_kernels.cu
+7
-5
@@ -109,7 +109,8 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
109
template <class Type, bool UE8M0_SF = false>
110
__global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
111
cvt_fp16_to_fp4_sf_major(int32_t numRows, int32_t numCols,
112
- int32_t sf_n_unpadded, Type const* __restrict__ in,
112
+ int32_t sf_n_unpadded, int32_t num_packed_cols,
113
+ Type const* __restrict__ in,
114
float const* __restrict__ SFScale,
115
uint32_t* __restrict__ out,
116
uint32_t* __restrict__ SFout) {
@@ -131,7 +132,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
132
// Iterate over all rows and cols including padded ones -
133
// ensures we visit every single scale factor address to initialize it.
134
for (int rowIdx = blockIdx.x; rowIdx < numRows; rowIdx += gridDim.x) {
134
- if (colIdx < sf_n_unpadded) {
135
+ if (colIdx < num_packed_cols) {
136
PackedVec in_vec;
137
int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
138
@@ -222,7 +223,8 @@ void scaled_fp4_quant_sm1xxa(torch::Tensor const& output,
223
reinterpret_cast<uint32_t*>(sf_out));
224
});
225
} else {
225
- int grid_y = vllm::div_round_up(sf_n_unpadded, static_cast<int>(block.x));
226
+ int num_packed_cols = n / CVT_FP4_ELTS_PER_THREAD;
227
+ int grid_y = vllm::div_round_up(num_packed_cols, static_cast<int>(block.x));
228
int grid_x = std::min(
229
m, std::max(1, (multiProcessorCount * numBlocksPerSM) / grid_y));
230
dim3 grid(grid_x, grid_y);
@@ -232,8 +234,8 @@ void scaled_fp4_quant_sm1xxa(torch::Tensor const& output,
234
auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
235
// NOTE: We don't support e8m0 scales at this moment.
236
vllm::cvt_fp16_to_fp4_sf_major<cuda_type, false>
235
- <<<grid, block, 0, stream>>>(m, n, sf_n_unpadded, input_ptr,
236
- input_sf_ptr,
237
+ <<<grid, block, 0, stream>>>(m, n, sf_n_unpadded, num_packed_cols,
238
+ input_ptr, input_sf_ptr,
239
reinterpret_cast<uint32_t*>(output_ptr),
240
reinterpret_cast<uint32_t*>(sf_out));
241
});