A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix][ROCm][Build] Fix build regression on ROCm (#21393)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
Jul 22, 2025 at 23:27 UTC
3ec7170ff191a89eb494677a04013a0872fe84fe
3 files changed
+16
-16
CMakeLists.txt
+2
-2
@@ -245,7 +245,6 @@ set(VLLM_EXT_SRC
245
"csrc/quantization/gptq/q_gemm.cu"
246
"csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
247
"csrc/quantization/fp8/common.cu"
248
- "csrc/quantization/fp8/per_token_group_quant.cu"
248
"csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu"
249
"csrc/quantization/gguf/gguf_kernel.cu"
250
"csrc/quantization/activation_kernels.cu"
@@ -297,7 +296,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
296
"csrc/quantization/fp4/nvfp4_blockwise_moe_kernel.cu"
297
"csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
298
"csrc/cutlass_extensions/common.cpp"
300
- "csrc/attention/mla/cutlass_mla_entry.cu")
299
+ "csrc/attention/mla/cutlass_mla_entry.cu"
300
+ "csrc/quantization/fp8/per_token_group_quant.cu")
301
302
set_gencode_flags_for_srcs(
303
SRCS "${VLLM_EXT_SRC}"
csrc/ops.h
+5
-5
@@ -287,6 +287,11 @@ void scaled_fp4_experts_quant(
287
torch::Tensor const& input, torch::Tensor const& input_global_scale,
288
torch::Tensor const& input_offset_by_experts,
289
torch::Tensor const& output_scale_offset_by_experts);
290
+
291
+void per_token_group_quant_fp8(const torch::Tensor& input,
292
+ torch::Tensor& output_q, torch::Tensor& output_s,
293
+ int64_t group_size, double eps, double fp8_min,
294
+ double fp8_max, bool scale_ue8m0);
295
#endif
296
297
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
@@ -297,11 +302,6 @@ void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
302
torch::Tensor& scales,
303
std::optional<torch::Tensor> const& azp);
304
300
-void per_token_group_quant_fp8(const torch::Tensor& input,
301
- torch::Tensor& output_q, torch::Tensor& output_s,
302
- int64_t group_size, double eps, double fp8_min,
303
- double fp8_max, bool scale_ue8m0);
304
-
305
torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
306
torch::Tensor b_gptq_qzeros,
307
torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
csrc/torch_bindings.cpp
+9
-9
@@ -601,15 +601,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
601
ops.impl("dynamic_scaled_int8_quant", torch::kCUDA,
602
&dynamic_scaled_int8_quant);
603
604
- // Compute per-token-group FP8 quantized tensor and scaling factor.
605
- ops.def(
606
- "per_token_group_fp8_quant(Tensor input, Tensor! output_q, Tensor! "
607
- "output_s, "
608
- "int group_size, float eps, float fp8_min, float fp8_max, bool "
609
- "scale_ue8m0) -> ()");
610
- ops.impl("per_token_group_fp8_quant", torch::kCUDA,
611
- &per_token_group_quant_fp8);
612
-
604
// Mamba selective scan kernel
605
ops.def(
606
"selective_scan_fwd(Tensor! u, Tensor! delta,"
@@ -624,6 +615,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
615
ops.impl("selective_scan_fwd", torch::kCUDA, &selective_scan_fwd);
616
617
#ifndef USE_ROCM
618
+ // Compute per-token-group FP8 quantized tensor and scaling factor.
619
+ ops.def(
620
+ "per_token_group_fp8_quant(Tensor input, Tensor! output_q, Tensor! "
621
+ "output_s, "
622
+ "int group_size, float eps, float fp8_min, float fp8_max, bool "
623
+ "scale_ue8m0) -> ()");
624
+ ops.impl("per_token_group_fp8_quant", torch::kCUDA,
625
+ &per_token_group_quant_fp8);
626
+
627
// reorder weight for AllSpark Ampere W8A16 Fused Gemm kernel
628
ops.def(
629
"rearrange_kn_weight_as_n32k16_order(Tensor b_qweight, Tensor b_scales, "