A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 23:45 UTC
[Bugfix] Guard mxfp4_experts_quant bindings on ENABLE_NVFP4_SM100 (#40191)
Signed-off-by: ultranationalism <www913363043@gmail.com> Signed-off-by: mgoin <mike.goin12@gmail.com> Co-authored-by: mgoin <mike.goin12@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
mysterious hhhh committed
Apr 19, 2026 at 04:58 UTC
d0359f3e0401f3ccb3b8fb66658908c08d265a44
3 files changed
+12
-20
csrc/libtorch_stable/ops.h
-14
@@ -134,20 +134,6 @@ void silu_and_mul_nvfp4_quant(torch::stable::Tensor& out,
134
torch::stable::Tensor& input,
135
torch::stable::Tensor& input_global_scale);
136
137
-void mxfp4_experts_quant(
138
- torch::stable::Tensor& output, torch::stable::Tensor& output_scale,
139
- torch::stable::Tensor const& input,
140
- torch::stable::Tensor const& input_offset_by_experts,
141
- torch::stable::Tensor const& output_scale_offset_by_experts,
142
- int64_t n_experts);
143
-
144
-void silu_and_mul_mxfp4_experts_quant(
145
- torch::stable::Tensor& output, torch::stable::Tensor& output_scale,
146
- torch::stable::Tensor const& input,
147
- torch::stable::Tensor const& input_offset_by_experts,
148
- torch::stable::Tensor const& output_scale_offset_by_experts,
149
- int64_t n_experts);
150
-
137
void cutlass_mxfp4_group_mm(torch::stable::Tensor& output,
138
const torch::stable::Tensor& a,
139
const torch::stable::Tensor& b,
csrc/libtorch_stable/quantization/fp4/mxfp4_experts_quant.cu
+10
@@ -23,6 +23,7 @@
23
#include <cuda_runtime.h>
24
#include <cuda_fp8.h>
25
26
+#include <torch/csrc/stable/library.h>
27
#include <torch/csrc/stable/tensor.h>
28
#include "libtorch_stable/torch_utils.h"
29
#include "libtorch_stable/dispatch_utils.h"
@@ -420,3 +421,12 @@ void silu_and_mul_mxfp4_experts_quant(
421
stream);
422
});
423
}
424
+
425
+// Registered here (not torch_bindings.cpp) because VLLM_GPU_FLAGS is applied
426
+// only under COMPILE_LANGUAGE:CUDA, so ENABLE_NVFP4_SM100 is invisible to
427
+// .cpp files and cannot gate the registration from there.
428
+STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
429
+ m.impl("mxfp4_experts_quant", TORCH_BOX(&mxfp4_experts_quant));
430
+ m.impl("silu_and_mul_mxfp4_experts_quant",
431
+ TORCH_BOX(&silu_and_mul_mxfp4_experts_quant));
432
+}
csrc/libtorch_stable/torch_bindings.cpp
+2
-6
@@ -252,12 +252,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
252
ops.impl("silu_and_mul_scaled_fp4_experts_quant",
253
TORCH_BOX(&silu_and_mul_scaled_fp4_experts_quant));
254
ops.impl("silu_and_mul_nvfp4_quant", TORCH_BOX(&silu_and_mul_nvfp4_quant));
255
- ops.impl("mxfp4_experts_quant", TORCH_BOX(&mxfp4_experts_quant));
256
- ops.impl("silu_and_mul_mxfp4_experts_quant",
257
- TORCH_BOX(&silu_and_mul_mxfp4_experts_quant));
258
-
259
- // W4A8 ops: impl registrations are in the source files
260
- // (w4a8_mm_entry.cu and w4a8_grouped_mm_entry.cu)
255
+ // mxfp4_experts_quant: registered in mxfp4_experts_quant.cu (SM100 only).
256
+ // W4A8 ops: registered in w4a8_mm_entry.cu / w4a8_grouped_mm_entry.cu.
257
#endif
258
}
259